def parse_vote_data(self, data):
        scrutin = self.get_scrutin(data['scrutin_url'])
        if scrutin is None:
            logger.debug('Cannot import vote for unknown scrutin %s'
                         % data['scrutin_url'])
            return

        if 'parl_url' in data:
            repdesc = data['parl_url']
            depute = self.get_depute_by_url(data['parl_url'])
        else:
            repdesc = '%s %s' % (data['prenom'], data['nom'])
            depute = self.get_depute_by_name(data['prenom'], data['nom'])

        if depute is None:
            logger.debug('Cannot import vote by unknown rep %s' % repdesc)
            return

        if not data['division'].lower() in self.positions:
            logger.debug('Cannot import vote for invalid position %s'
                         % data['division'])
            return
        position = self.positions[data['division'].lower()]

        changed = False
        try:
            vote = Vote.objects.get(representative_id=depute,
                                    proposal_id=scrutin)
        except Vote.DoesNotExist:
            vote = Vote(representative_id=depute, proposal_id=scrutin)
            logger.debug('Created vote for rep %s on %s' % (depute, scrutin))
            changed = True

        if vote.position != position:
            logger.debug('Changed vote position to %s' % position)
            changed = True
            vote.position = position

        if changed:
            logger.debug('Updated vote for rep %s on %s' % (depute, scrutin))
            self.touched.append(scrutin)
            vote.save()
Example #2
0
    def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
        logger.info(u'Looking for votes in proposal {}'.format(proposal.title))

        for position in ('For', 'Abstain', 'Against'):
            for group_vote_data in data.get(position_map[position],
                                            {}).get('groups', {}):
                for vote_data in group_vote_data['votes']:
                    if not isinstance(vote_data, dict):
                        logger.error('Skipping vote data %s for proposal %s',
                                     vote_data, data['_id'])
                        continue

                    representative_pk = self.get_representative(vote_data)

                    if representative_pk is None:
                        logger.error('Could not find mep for %s', vote_data)
                        continue

                    changed = False
                    try:
                        vote = Vote.objects.get(
                            representative_id=representative_pk,
                            proposal_id=proposal.pk)
                    except Vote.DoesNotExist:
                        vote = Vote(proposal_id=proposal.pk,
                                    representative_id=representative_pk)
                        changed = True

                    if vote.position != position.lower():
                        changed = True
                        vote.position = position.lower()

                    if changed:
                        vote.save()
                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
                                     vote.pk, representative_pk,
                                     proposal.title, proposal.pk, position)
    def parse_proposal_data(self, proposal_data, dossier_pk):
        """Get or Create a proposal model from raw data"""
        proposal_display = "{} ({})".format(
            proposal_data["title"].encode("utf-8"), proposal_data.get("report", "").encode("utf-8")
        )

        if "issue_type" not in proposal_data.keys():
            logger.debug("This proposal data without issue_type: %s", proposal_data["epref"])
            return

        changed = False
        try:
            proposal = Proposal.objects.get(title=proposal_data["title"])
        except Proposal.DoesNotExist:
            proposal = Proposal(title=proposal_data["title"])
            changed = True

        data_map = dict(
            title=proposal_data["title"],
            datetime=_parse_date(proposal_data["ts"]),
            dossier_id=dossier_pk,
            reference=proposal_data.get("report"),
            kind=proposal_data.get("issue_type"),
        )

        for position in ("For", "Abstain", "Against"):
            position_data = proposal_data.get(position, {})
            position_total = position_data.get("total", 0)

            if isinstance(position_total, str) and position_total.isdigit():
                position_total = int(position_total)

            data_map["total_%s" % position.lower()] = position_total

        for key, value in data_map.items():
            if value != getattr(proposal, key, None):
                setattr(proposal, key, value)
                changed = True

        if changed:
            proposal.save()

        if self.should_skip(proposal_data):
            logger.debug("Skipping dossier %s", proposal_data.get("epref", proposal_data["title"]))
            return

        positions = ["For", "Abstain", "Against"]
        logger.info("Looking for votes in proposal {}".format(proposal_display))
        for position in positions:
            for group_vote_data in proposal_data.get(position, {}).get("groups", {}):
                for vote_data in group_vote_data["votes"]:
                    if not isinstance(vote_data, dict):
                        logger.error("Skipping vote data %s for proposal %s", vote_data, proposal_data["_id"])
                        continue

                    representative_pk = self.get_representative(vote_data)

                    if representative_pk is None:
                        logger.error("Could not find mep for %s", vote_data)
                        continue

                    representative_name = vote_data.get("orig", "")

                    changed = False
                    try:
                        vote = Vote.objects.get(representative_id=representative_pk, proposal_id=proposal.pk)
                    except Vote.DoesNotExist:
                        vote = Vote(proposal_id=proposal.pk, representative_id=representative_pk)
                        changed = True

                    if vote.position != position.lower():
                        changed = True
                        vote.position = position.lower()

                    if vote.representative_name != representative_name:
                        changed = True
                        vote.representative_name = representative_name

                    if changed:
                        vote.save()
                        logger.debug(
                            "Save vote %s for MEP %s on %s #%s to %s",
                            vote.pk,
                            representative_pk,
                            proposal_data["title"],
                            proposal.pk,
                            position,
                        )

        return proposal
    def parse_proposal_data(self, proposal_data, dossier_pk):
        """Get or Create a proposal model from raw data"""
        proposal_display = '{} ({})'.format(
            proposal_data['title'].encode('utf-8'),
            proposal_data.get('report', '').encode('utf-8'))

        if 'issue_type' not in proposal_data.keys():
            logger.debug('This proposal data without issue_type: %s',
                         proposal_data['epref'])
            return

        changed = False
        try:
            proposal = Proposal.objects.get(title=proposal_data['title'])
        except Proposal.DoesNotExist:
            proposal = Proposal(title=proposal_data['title'])
            changed = True

        data_map = dict(title=proposal_data['title'],
                        datetime=_parse_date(proposal_data['ts']),
                        dossier_id=dossier_pk,
                        reference=proposal_data.get('report'),
                        kind=proposal_data.get('issue_type'))

        for position in ('For', 'Abstain', 'Against'):
            position_data = proposal_data.get(position, {})
            position_total = position_data.get('total', 0)

            if isinstance(position_total, str) and position_total.isdigit():
                position_total = int(position_total)

            data_map['total_%s' % position.lower()] = position_total

        for key, value in data_map.items():
            if value != getattr(proposal, key, None):
                setattr(proposal, key, value)
                changed = True

        if changed:
            proposal.save()

        if self.should_skip(proposal_data):
            logger.debug('Skipping dossier %s',
                         proposal_data.get('epref', proposal_data['title']))
            return

        positions = ['For', 'Abstain', 'Against']
        logger.info(
            'Looking for votes in proposal {}'.format(proposal_display))
        for position in positions:
            for group_vote_data in proposal_data.get(position,
                                                     {}).get('groups', {}):
                for vote_data in group_vote_data['votes']:
                    if not isinstance(vote_data, dict):
                        logger.error('Skipping vote data %s for proposal %s',
                                     vote_data, proposal_data['_id'])
                        continue

                    representative_pk = self.get_representative(vote_data)

                    if representative_pk is None:
                        logger.error('Could not find mep for %s', vote_data)
                        continue

                    representative_name = vote_data.get('orig', '')

                    changed = False
                    try:
                        vote = Vote.objects.get(
                            representative_id=representative_pk,
                            proposal_id=proposal.pk)
                    except Vote.DoesNotExist:
                        vote = Vote(proposal_id=proposal.pk,
                                    representative_id=representative_pk)
                        changed = True

                    if vote.position != position.lower():
                        changed = True
                        vote.position = position.lower()

                    if vote.representative_name != representative_name:
                        changed = True
                        vote.representative_name = representative_name

                    if changed:
                        vote.save()
                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
                                     vote.pk, representative_pk,
                                     proposal_data['title'], proposal.pk,
                                     position)

        return proposal
    def parse_proposal_data(self, proposal_data, dossier_pk):
        """Get or Create a proposal model from raw data"""
        proposal_display = '{} ({})'.format(proposal_data['title'].encode(
            'utf-8'), proposal_data.get('report', '').encode('utf-8'))

        if 'issue_type' not in proposal_data.keys():
            logger.debug('This proposal data without issue_type: %s',
                         proposal_data['epref'])
            return

        changed = False
        try:
            proposal = Proposal.objects.get(title=proposal_data['title'])
        except Proposal.DoesNotExist:
            proposal = Proposal(title=proposal_data['title'])
            changed = True

        data_map = dict(
            title=proposal_data['title'],
            datetime=_parse_date(proposal_data['ts']),
            dossier_id=dossier_pk,
            reference=proposal_data.get('report'),
            kind=proposal_data.get('issue_type')
        )

        for position in ('For', 'Abstain', 'Against'):
            position_data = proposal_data.get(position, {})
            position_total = position_data.get('total', 0)

            if isinstance(position_total, str) and position_total.isdigit():
                position_total = int(position_total)

            data_map['total_%s' % position.lower()] = position_total

        for key, value in data_map.items():
            if value != getattr(proposal, key, None):
                setattr(proposal, key, value)
                changed = True

        if changed:
            proposal.save()

        responses = vote_pre_import.send(sender=self, vote_data=proposal_data)

        for receiver, response in responses:
            if response is False:
                logger.debug(
                    'Skipping dossier %s', proposal_data.get(
                        'epref', proposal_data['title']))
                return

        positions = ['For', 'Abstain', 'Against']
        logger.info(
            'Looking for votes in proposal {}'.format(proposal_display))
        for position in positions:
            for group_vote_data in proposal_data.get(
                    position,
                    {}).get(
                    'groups',
                    {}):
                for vote_data in group_vote_data['votes']:
                    if not isinstance(vote_data, dict):
                        logger.error('Skipping vote data %s for proposal %s',
                                     vote_data, proposal_data['_id'])
                        continue

                    representative_pk = self.get_representative(vote_data)

                    if representative_pk is None:
                        logger.error('Could not find mep for %s', vote_data)
                        continue

                    representative_name = vote_data.get('orig', '')

                    changed = False
                    try:
                        vote = Vote.objects.get(
                            representative_id=representative_pk,
                            proposal_id=proposal.pk)
                    except Vote.DoesNotExist:
                        vote = Vote(proposal_id=proposal.pk,
                            representative_id=representative_pk)
                        changed = True

                    if vote.position != position.lower():
                        changed = True
                        vote.position = position.lower()

                    if vote.representative_name != representative_name:
                        changed = True
                        vote.representative_name = representative_name

                    if changed:
                        vote.save()
                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
                            vote.pk, representative_pk, proposal_data['title'],
                            proposal.pk, position)

        return proposal