def handle(self, *args, **options):
self.out_csv = BufferDictWriter(self.fieldnames)
self.out_csv.writeheader()
qs = OfficialDocument.objects.filter(
election__election_date=options["election_date"])
for document in qs:
self.check_doc(document)
self.stdout.write(self.out_csv.output)
def list_to_csv(candidates_list, group_by_post=False):
from .election_specific import EXTRA_CSV_ROW_FIELDS
csv_fields = CSV_ROW_FIELDS + EXTRA_CSV_ROW_FIELDS
writer = BufferDictWriter(fieldnames=csv_fields)
writer.writeheader()
if group_by_post:
sorted_rows = sorted(candidates_list, key=_candidate_sort_by_post_key)
else:
sorted_rows = sorted(candidates_list, key=_candidate_sort_by_name_key)
for row in sorted_rows:
writer.writerow(row)
return writer.output
def handle(self, *args, **options):
fieldnames = (
'party_id',
'party_name',
'party_description',
)
out_csv = BufferDictWriter(fieldnames)
out_csv.writeheader()
for org in Organization.objects.filter(classification='Party'):
try:
party_id = org.identifiers.get(
scheme='electoral-commission').identifier
except Identifier.DoesNotExist:
party_id = org.identifiers.get(
scheme='popit-organization').identifier
out_dict = {
'party_id': party_id,
'party_name': org.name,
'party_description': '',
}
out_csv.writerow(out_dict)
for desc in org.other_names.all():
out_dict['party_description'] = desc.name
out_csv.writerow(out_dict)
self.stdout.write(out_csv.output)
def handle(self, *args, **options):
import time
start_time = time.time()
fieldnames = (
'election_id',
'division_id',
'division_name',
'candidate_id',
'candidate_name',
'candidate_other_names',
'party_id',
'party_name',
'document_id',
)
out_csv = BufferDictWriter(fieldnames)
out_csv.writeheader()
documents = OfficialDocument.objects.all().order_by('election', 'post')
for document in documents:
document_memberships = Membership.objects.filter(
post=document.post, election=document.election).select_related(
'base',
'on_behalf_of',
'post__extra',
'post',
'person',
)
out_dict = {
'election_id': document.election.slug,
'division_id': document.post.extra.slug,
'division_name': document.post.extra.short_label,
'document_id': document.pk,
}
for membership in document_memberships:
other_names = "|".join(
[o.name for o in membership.base.person.other_names.all()])
party = membership.base.on_behalf_of
try:
party_id = party.identifiers.get(
scheme='electoral-commission').identifier
except Identifier.DoesNotExist:
party_id = party.identifiers.get(
scheme='popit-organization').identifier
out_dict.update({
'candidate_id': membership.base.person.id,
'candidate_name': membership.base.person.name,
'candidate_other_names': other_names,
'party_id': party_id,
'party_name': party.name,
})
out_csv.writerow(out_dict)
else:
out_csv.writerow(out_dict)
self.stdout.write(out_csv.output)
def handle(self, *args, **options):
out_csv = BufferDictWriter(self.fieldnames)
out_csv.writeheader()
qs = PostExtraElection.objects.filter(
election__election_date=options['election_date']).select_related(
'election', 'postextra',
'postextra__base').order_by('election__slug')
if not options['non_current']:
qs = qs.filter(election__current=True)
for pee in qs:
row = {
'ballot_paper_id': pee.ballot_paper_id,
'Election name': pee.election.name,
'Area name': pee.postextra.base.label,
'Council webpage likely to link to SOPN': '',
'Link to PDF': '',
'Notes': '',
}
out_csv.writerow(row)
self.stdout.write(out_csv.output)
def handle(self, *args, **options):
import time
start_time = time.time()
fieldnames = (
"election_id",
"division_id",
"division_name",
"candidate_id",
"candidate_name",
"candidate_other_names",
"party_id",
"party_name",
"document_id",
)
out_csv = BufferDictWriter(fieldnames)
out_csv.writeheader()
documents = OfficialDocument.objects.all().order_by("election", "post")
for document in documents:
document_memberships = Membership.objects.filter(
post=document.post, election=document.election).select_related(
"base", "party", "post", "person")
out_dict = {
"election_id": document.election.slug,
"division_id": document.post.slug,
"division_name": document.post.short_label,
"document_id": document.pk,
}
for membership in document_memberships:
other_names = "|".join(
[o.name for o in membership.base.person.other_names.all()])
party = membership.party
out_dict.update({
"candidate_id": membership.base.person.id,
"candidate_name": membership.base.person.name,
"candidate_other_names": other_names,
"party_id": party.ec_id,
"party_name": party.name,
})
out_csv.writerow(out_dict)
else:
out_csv.writerow(out_dict)
self.stdout.write(out_csv.output)
def list_to_csv(membership_list):
csv_fields = settings.CSV_ROW_FIELDS
writer = BufferDictWriter(fieldnames=csv_fields)
writer.writeheader()
for row in sorted(membership_list, key=lambda d: (d["election_date"])):
writer.writerow(row)
return writer.output
def list_to_csv(candidates_list, group_by_post=False):
from .election_specific import EXTRA_CSV_ROW_FIELDS
csv_fields = CSV_ROW_FIELDS + EXTRA_CSV_ROW_FIELDS
writer = BufferDictWriter(fieldnames=csv_fields)
writer.writeheader()
if group_by_post:
sorted_rows = sorted(candidates_list, key=_candidate_sort_by_post_key)
else:
sorted_rows = sorted(candidates_list, key=_candidate_sort_by_name_key)
for row in sorted_rows:
writer.writerow(row)
return writer.output
def handle(self, **options):
date = options['election_date']
qs = ResultSet.objects.filter(
post_election__election__election_date=date).select_related(
'post_election',
'post_election__election',
).prefetch_related(
Prefetch(
'post_election__membershipextra_set',
MembershipExtra.objects.select_related(
'base',
'base__person',
'base__on_behalf_of',
)))
csv_out = BufferDictWriter(fieldnames=self.FIELDNAMES)
csv_out.writeheader()
for result in qs:
row_base = {
'election_id': result.post_election.election.slug,
'ballot_paper_id': result.post_election.ballot_paper_id,
}
for membership in result.post_election.membershipextra_set.all():
if not hasattr(membership.base, 'result'):
continue
row = row_base
party = membership.base.on_behalf_of
try:
if party.name == "Independent":
party_id = "ynmp-party:2"
else:
party_id = party.identifiers.filter(
scheme='electoral-commission').get().identifier
except:
party_id = ""
row['party_id'] = party_id
row['party_name'] = membership.base.on_behalf_of.name
row['person_id'] = membership.base.person.pk
row['person_name'] = membership.base.person.name
row['ballots_cast'] = membership.base.result.num_ballots
row['is_winner'] = membership.base.result.is_winner
csv_out.writerow(row)
self.stdout.write(csv_out.output)
def handle(self, *args, **options):
out_csv = BufferDictWriter(self.fieldnames)
out_csv.writeheader()
qs = (Ballot.objects.filter(
election__election_date=options["election_date"]).select_related(
"election", "postextra", "post").order_by("election__slug"))
if not options["non_current"]:
qs = qs.filter(election__current=True)
for ballot in qs:
row = {
"ballot_paper_id": ballot.ballot_paper_id,
"Election name": ballot.election.name,
"Area name": ballot.post.label,
"Council webpage likely to link to SOPN": "",
"Link to PDF": "",
"Notes": "",
}
out_csv.writerow(row)
self.stdout.write(out_csv.output)
def handle(self, **options):
date = options["election_date"]
format = options["format"]
directory_path = "csv-archives"
self.storage = DefaultStorage()
output_filename = "{}/results-{}.{}".format(directory_path, date,
format)
qs = (ResultSet.objects.filter(
ballot__election__election_date=date).select_related(
"ballot", "ballot__election").prefetch_related(
Prefetch(
"ballot__membership_set",
Membership.objects.select_related("person", "party"),
)))
out_data = []
for result in qs:
for membership in result.ballot.membership_set.all():
if not hasattr(membership, "result"):
continue
row = {
"election_id": result.ballot.election.slug,
"ballot_paper_id": result.ballot.ballot_paper_id,
"turnout": result.num_turnout_reported,
"spoilt_ballots": result.num_spoilt_ballots,
"source": result.source,
}
party = membership.party
try:
if party.name == "Independent":
party_id = "ynmp-party:2"
else:
party_id = (party.identifiers.filter(
scheme="electoral-commission").get().identifier)
except:
party_id = ""
row["party_id"] = party_id
row["party_name"] = party.name
row["person_id"] = membership.person.pk
row["person_name"] = membership.person.name
row["ballots_cast"] = membership.result.num_ballots
row["is_winner"] = membership.result.is_winner
out_data.append(row)
if format == "csv":
csv_out = BufferDictWriter(fieldnames=self.FIELDNAMES)
csv_out.writeheader()
for row in out_data:
csv_out.writerow(row)
out_string = csv_out.output
else:
json_data = defaultdict(dict)
for person in out_data:
election_dict = json_data[person["ballot_paper_id"]]
election_dict["turnout"] = person["turnout"]
election_dict["spoilt_ballots"] = person["spoilt_ballots"]
election_dict["source"] = person["source"]
if "candidates" not in election_dict:
election_dict["candidates"] = []
election_dict["candidates"].append({
"person_name":
person["person_name"],
"person_id":
person["person_id"],
"party_name":
person["party_name"],
"party_id":
person["party_id"],
"ballots_cast":
person["ballots_cast"],
"is_winner":
person["is_winner"],
})
election_dict["candidates"] = sorted(
election_dict["candidates"],
key=lambda p: p["ballots_cast"],
reverse=True,
)
out_string = json.dumps(json_data, indent=4)
self.storage.save(output_filename,
ContentFile(out_string.encode("utf8")))
class Command(BaseCommand):
help = "Check the hash of a document against the source"
fieldnames = [
"ballot_paper_id",
"Election name",
"Area name",
"Source URL",
"remote_hash",
"local_hash",
"status_code",
"notes",
]
url_info_cache = {}
def add_arguments(self, parser):
parser.add_argument("--election-date", action="store", required=True)
def handle(self, *args, **options):
self.out_csv = BufferDictWriter(self.fieldnames)
self.out_csv.writeheader()
qs = OfficialDocument.objects.filter(
election__election_date=options["election_date"])
for document in qs:
self.check_doc(document)
self.stdout.write(self.out_csv.output)
def get_hash(self, sopn_file):
md5 = hashlib.md5()
md5.update(sopn_file)
return md5.hexdigest()
def check_doc(self, doc):
line = {
"ballot_paper_id": doc.ballot.ballot_paper_id,
"Election name": doc.ballot.election.name,
"Area name": doc.ballot.post.label,
"Source URL": doc.source_url,
}
cache = self.url_info_cache.get(doc.source_url, {})
try:
if not cache:
req = requests.get(doc.source_url, stream=True)
status_code = req.status_code
cache = {"status_code": status_code}
if status_code != 200:
cache["notes"] = "Remote file missing!"
else:
cache["remote_hash"] = self.get_hash(req.raw.read())
cache["local_hash"] = self.get_hash(
doc.uploaded_file.file.read())
if not cache["remote_hash"] == cache["local_hash"]:
cache["notes"] = "File hash mismatch!"
line.update(cache)
self.url_info_cache[doc.source_url] = cache
except requests.exceptions.RequestException as e:
cache["notes"] = e
self.out_csv.writerow(line)
def handle(self, **options):
date = options['election_date']
format = options['format']
qs = ResultSet.objects.filter(
post_election__election__election_date=date).select_related(
'post_election',
'post_election__election',
).prefetch_related(
Prefetch(
'post_election__membership_set',
Membership.objects.select_related(
'person',
'on_behalf_of',
)))
out_data = []
for result in qs:
for membership in result.post_election.membership_set.all():
if not hasattr(membership, 'result'):
continue
row = {
'election_id': result.post_election.election.slug,
'ballot_paper_id': result.post_election.ballot_paper_id,
'turnout': result.num_turnout_reported,
'spoilt_ballots': result.num_spoilt_ballots,
'source': result.source,
}
party = membership.on_behalf_of
try:
if party.name == "Independent":
party_id = "ynmp-party:2"
else:
party_id = party.identifiers.filter(
scheme='electoral-commission').get().identifier
except:
party_id = ""
row['party_id'] = party_id
row['party_name'] = membership.on_behalf_of.name
row['person_id'] = membership.person.pk
row['person_name'] = membership.person.name
row['ballots_cast'] = membership.result.num_ballots
row['is_winner'] = membership.result.is_winner
out_data.append(row)
if format == "csv":
csv_out = BufferDictWriter(fieldnames=self.FIELDNAMES)
csv_out.writeheader()
for row in out_data:
csv_out.writerow(row)
self.stdout.write(csv_out.output)
else:
json_data = defaultdict(dict)
for person in out_data:
election_dict = json_data[person['ballot_paper_id']]
election_dict['turnout'] = person['turnout']
election_dict['spoilt_ballots'] = person['spoilt_ballots']
election_dict['source'] = person['source']
if 'candidates' not in election_dict:
election_dict['candidates'] = []
election_dict['candidates'].append({
'person_name':
person['person_name'],
'person_id':
person['person_id'],
'party_name':
person['party_name'],
'party_id':
person['party_id'],
'ballots_cast':
person['ballots_cast'],
'is_winner':
person['is_winner']
})
election_dict['candidates'] = sorted(
election_dict['candidates'],
key=lambda p: p['ballots_cast'],
reverse=True)
self.stdout.write(json.dumps(json_data, indent=4))
class Command(BaseCommand):
help = "Check the hash of a document against the source"
fieldnames = [
'ballot_paper_id',
'Election name',
'Area name',
'Source URL',
'remote_hash',
'local_hash',
'status_code',
'notes',
]
url_info_cache = {}
def add_arguments(self, parser):
parser.add_argument('--election-date', action='store', required=True)
def handle(self, *args, **options):
self.out_csv = BufferDictWriter(self.fieldnames)
self.out_csv.writeheader()
qs = OfficialDocument.objects.filter(
election__election_date=options['election_date'])
for document in qs:
self.check_doc(document)
self.stdout.write(self.out_csv.output)
def get_hash(self, sopn_file):
md5 = hashlib.md5()
md5.update(sopn_file)
return md5.hexdigest()
def check_doc(self, doc):
line = {
'ballot_paper_id': doc.post_election.ballot_paper_id,
'Election name': doc.post_election.election.name,
'Area name': doc.post_election.postextra.base.label,
'Source URL': doc.source_url,
}
cache = self.url_info_cache.get(doc.source_url, {})
try:
if not cache:
req = requests.get(doc.source_url, stream=True)
status_code = req.status_code
cache = {
'status_code': status_code,
}
if status_code != 200:
cache['notes'] = "Remote file missing!"
else:
cache['remote_hash'] = self.get_hash(req.raw.read())
cache['local_hash'] = self.get_hash(
doc.uploaded_file.file.read())
if not cache['remote_hash'] == cache['local_hash']:
cache['notes'] = 'File hash mismatch!'
line.update(cache)
self.url_info_cache[doc.source_url] = cache
except requests.exceptions.RequestException as e:
cache['notes'] = e
self.out_csv.writerow(line)
def handle(self, **options):
date = options["election_date"]
format = options["format"]
qs = (
ResultSet.objects.filter(
post_election__election__election_date=date
)
.select_related("post_election", "post_election__election")
.prefetch_related(
Prefetch(
"post_election__membership_set",
Membership.objects.select_related("person", "on_behalf_of"),
)
)
)
out_data = []
for result in qs:
for membership in result.post_election.membership_set.all():
if not hasattr(membership, "result"):
continue
row = {
"election_id": result.post_election.election.slug,
"ballot_paper_id": result.post_election.ballot_paper_id,
"turnout": result.num_turnout_reported,
"spoilt_ballots": result.num_spoilt_ballots,
"source": result.source,
}
party = membership.on_behalf_of
try:
if party.name == "Independent":
party_id = "ynmp-party:2"
else:
party_id = (
party.identifiers.filter(
scheme="electoral-commission"
)
.get()
.identifier
)
except:
party_id = ""
row["party_id"] = party_id
row["party_name"] = membership.on_behalf_of.name
row["person_id"] = membership.person.pk
row["person_name"] = membership.person.name
row["ballots_cast"] = membership.result.num_ballots
row["is_winner"] = membership.result.is_winner
out_data.append(row)
if format == "csv":
csv_out = BufferDictWriter(fieldnames=self.FIELDNAMES)
csv_out.writeheader()
for row in out_data:
csv_out.writerow(row)
self.stdout.write(csv_out.output)
else:
json_data = defaultdict(dict)
for person in out_data:
election_dict = json_data[person["ballot_paper_id"]]
election_dict["turnout"] = person["turnout"]
election_dict["spoilt_ballots"] = person["spoilt_ballots"]
election_dict["source"] = person["source"]
if "candidates" not in election_dict:
election_dict["candidates"] = []
election_dict["candidates"].append(
{
"person_name": person["person_name"],
"person_id": person["person_id"],
"party_name": person["party_name"],
"party_id": person["party_id"],
"ballots_cast": person["ballots_cast"],
"is_winner": person["is_winner"],
}
)
election_dict["candidates"] = sorted(
election_dict["candidates"],
key=lambda p: p["ballots_cast"],
reverse=True,
)
self.stdout.write(json.dumps(json_data, indent=4))