def parse_committee_vote_data(self, proposal_data, dossier_pk):
title = u'%s vote on %s' % (proposal_data['committee'],
proposal_data['doc'])
changed = False
try:
proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk)
except Proposal.DoesNotExist:
proposal = Proposal(title=title, dossier_id=dossier_pk)
changed = True
try:
ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1)
except:
logger.debug(u'Cannot find proposal reference for %s' % title)
return
data_map = dict(datetime=_parse_date(proposal_data['ts']),
reference=ref,
kind='committee-vote')
position_map = {'For': '+', 'Against': '-', 'Abstain': '0'}
data_map.update(self.parse_proposal_totals(proposal_data,
position_map))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug(u'Skipping votes for dossier %s',
proposal_data.get('epref', title))
return
self.parse_proposal_votes(proposal, proposal_data, position_map)
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data['title'])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data['title'])
changed = True
data_map = dict(title=proposal_data['title'],
datetime=_parse_date(proposal_data['ts']),
dossier_id=dossier_pk,
reference=proposal_data.get('report'),
kind=proposal_data.get('issue_type'))
data_map.update(self.parse_proposal_totals(proposal_data))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug(u'Skipping votes for dossier %s',
proposal_data.get('epref', proposal_data['title']))
return
self.parse_proposal_votes(proposal, proposal_data)
return proposal
def parse_scrutin_data(self, data):
ref = data['url']
if 'dossier_url' not in data:
logger.debug('Cannot create proposal without dossier')
return
dossier = self.get_dossier(data['dossier_url'])
if dossier is None:
logger.debug('Cannot create proposal for unknown dossier %s'
% data['dossier_url'])
return
changed = False
try:
proposal = Proposal.objects.get(reference=ref)
except Proposal.DoesNotExist:
proposal = Proposal(reference=ref, total_for=0, total_against=0,
total_abstain=0)
logger.debug('Created proposal %s' % ref)
changed = True
values = dict(
title=_get_unique_title(proposal.pk, data["objet"]),
datetime=_parse_date(data["date"]),
dossier_id=dossier.pk,
kind='dossier'
)
for key, value in values.items():
if value != getattr(proposal, key, None):
logger.debug('Changed proposal %s to %s' % (key, value))
setattr(proposal, key, value)
changed = True
if changed:
logger.debug('Updated proposal %s' % ref)
proposal.save()
def parse_scrutin_data(self, data):
ref = data['url']
if 'dossier_url' not in data:
logger.debug('Cannot create proposal without dossier')
return
dossier = self.get_dossier(data['dossier_url'])
if dossier is None:
logger.debug('Cannot create proposal for unknown dossier %s'
% data['dossier_url'])
return
changed = False
try:
proposal = Proposal.objects.get(reference=ref)
except Proposal.DoesNotExist:
proposal = Proposal(reference=ref, total_for=0, total_against=0,
total_abstain=0)
logger.debug('Created proposal %s' % ref)
changed = True
values = dict(
title=_get_unique_title(proposal.pk, data["objet"]),
datetime=_parse_date(data["date"]),
dossier_id=dossier.pk,
kind='dossier'
)
for key, value in values.items():
if value != getattr(proposal, key, None):
logger.debug('Changed proposal %s to %s' % (key, value))
setattr(proposal, key, value)
changed = True
if changed:
logger.debug('Updated proposal %s' % ref)
proposal.save()
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = "{} ({})".format(
proposal_data["title"].encode("utf-8"), proposal_data.get("report", "").encode("utf-8")
)
if "issue_type" not in proposal_data.keys():
logger.debug("This proposal data without issue_type: %s", proposal_data["epref"])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data["title"])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data["title"])
changed = True
data_map = dict(
title=proposal_data["title"],
datetime=_parse_date(proposal_data["ts"]),
dossier_id=dossier_pk,
reference=proposal_data.get("report"),
kind=proposal_data.get("issue_type"),
)
for position in ("For", "Abstain", "Against"):
position_data = proposal_data.get(position, {})
position_total = position_data.get("total", 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map["total_%s" % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug("Skipping dossier %s", proposal_data.get("epref", proposal_data["title"]))
return
positions = ["For", "Abstain", "Against"]
logger.info("Looking for votes in proposal {}".format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(position, {}).get("groups", {}):
for vote_data in group_vote_data["votes"]:
if not isinstance(vote_data, dict):
logger.error("Skipping vote data %s for proposal %s", vote_data, proposal_data["_id"])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error("Could not find mep for %s", vote_data)
continue
representative_name = vote_data.get("orig", "")
changed = False
try:
vote = Vote.objects.get(representative_id=representative_pk, proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk, representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug(
"Save vote %s for MEP %s on %s #%s to %s",
vote.pk,
representative_pk,
proposal_data["title"],
proposal.pk,
position,
)
return proposal
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(
proposal_data['title'].encode('utf-8'),
proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data['title'])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data['title'])
changed = True
data_map = dict(title=proposal_data['title'],
datetime=_parse_date(proposal_data['ts']),
dossier_id=dossier_pk,
reference=proposal_data.get('report'),
kind=proposal_data.get('issue_type'))
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug('Skipping dossier %s',
proposal_data.get('epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(position,
{}).get('groups', {}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk,
proposal_data['title'], proposal.pk,
position)
return proposal
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(proposal_data['title'].encode(
'utf-8'), proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data['title'])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data['title'])
changed = True
data_map = dict(
title=proposal_data['title'],
datetime=_parse_date(proposal_data['ts']),
dossier_id=dossier_pk,
reference=proposal_data.get('report'),
kind=proposal_data.get('issue_type')
)
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
for receiver, response in responses:
if response is False:
logger.debug(
'Skipping dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(
position,
{}).get(
'groups',
{}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, proposal_data['title'],
proposal.pk, position)
return proposal
