def parse_vote_data(self, data):
scrutin = self.get_scrutin(data['scrutin_url'])
if scrutin is None:
logger.debug('Cannot import vote for unknown scrutin %s'
% data['scrutin_url'])
return
if 'parl_url' in data:
repdesc = data['parl_url']
depute = self.get_depute_by_url(data['parl_url'])
else:
repdesc = '%s %s' % (data['prenom'], data['nom'])
depute = self.get_depute_by_name(data['prenom'], data['nom'])
if depute is None:
logger.debug('Cannot import vote by unknown rep %s' % repdesc)
return
if not data['division'].lower() in self.positions:
logger.debug('Cannot import vote for invalid position %s'
% data['division'])
return
position = self.positions[data['division'].lower()]
changed = False
try:
vote = Vote.objects.get(representative_id=depute,
proposal_id=scrutin)
except Vote.DoesNotExist:
vote = Vote(representative_id=depute, proposal_id=scrutin)
logger.debug('Created vote for rep %s on %s' % (depute, scrutin))
changed = True
if vote.position != position:
logger.debug('Changed vote position to %s' % position)
changed = True
vote.position = position
if changed:
logger.debug('Updated vote for rep %s on %s' % (depute, scrutin))
self.touched.append(scrutin)
vote.save()
def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
logger.info(u'Looking for votes in proposal {}'.format(proposal.title))
for position in ('For', 'Abstain', 'Against'):
for group_vote_data in data.get(position_map[position],
{}).get('groups', {}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk,
proposal.title, proposal.pk, position)
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = "{} ({})".format(
proposal_data["title"].encode("utf-8"), proposal_data.get("report", "").encode("utf-8")
)
if "issue_type" not in proposal_data.keys():
logger.debug("This proposal data without issue_type: %s", proposal_data["epref"])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data["title"])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data["title"])
changed = True
data_map = dict(
title=proposal_data["title"],
datetime=_parse_date(proposal_data["ts"]),
dossier_id=dossier_pk,
reference=proposal_data.get("report"),
kind=proposal_data.get("issue_type"),
)
for position in ("For", "Abstain", "Against"):
position_data = proposal_data.get(position, {})
position_total = position_data.get("total", 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map["total_%s" % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug("Skipping dossier %s", proposal_data.get("epref", proposal_data["title"]))
return
positions = ["For", "Abstain", "Against"]
logger.info("Looking for votes in proposal {}".format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(position, {}).get("groups", {}):
for vote_data in group_vote_data["votes"]:
if not isinstance(vote_data, dict):
logger.error("Skipping vote data %s for proposal %s", vote_data, proposal_data["_id"])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error("Could not find mep for %s", vote_data)
continue
representative_name = vote_data.get("orig", "")
changed = False
try:
vote = Vote.objects.get(representative_id=representative_pk, proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk, representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug(
"Save vote %s for MEP %s on %s #%s to %s",
vote.pk,
representative_pk,
proposal_data["title"],
proposal.pk,
position,
)
return proposal
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(
proposal_data['title'].encode('utf-8'),
proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data['title'])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data['title'])
changed = True
data_map = dict(title=proposal_data['title'],
datetime=_parse_date(proposal_data['ts']),
dossier_id=dossier_pk,
reference=proposal_data.get('report'),
kind=proposal_data.get('issue_type'))
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug('Skipping dossier %s',
proposal_data.get('epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(position,
{}).get('groups', {}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk,
proposal_data['title'], proposal.pk,
position)
return proposal
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(proposal_data['title'].encode(
'utf-8'), proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data['title'])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data['title'])
changed = True
data_map = dict(
title=proposal_data['title'],
datetime=_parse_date(proposal_data['ts']),
dossier_id=dossier_pk,
reference=proposal_data.get('report'),
kind=proposal_data.get('issue_type')
)
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
for receiver, response in responses:
if response is False:
logger.debug(
'Skipping dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(
position,
{}).get(
'groups',
{}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, proposal_data['title'],
proposal.pk, position)
return proposal