def download_centers_csv(request):
"""View to handle 'download centers as CSV' link"""
response = prepare_csv_response("centers")
writer = UnicodeWriter(response)
# write header row
writer.writerow(CSV_FIELDS)
for center in RegistrationCenter.objects.all():
writer.writerow([format_field(center, field) for field in CSV_FIELDS])
return response
def generate_polling_metadata_csv():
election = Election.objects.get_most_current_election()
from rollgen.models import Station
stations = Station.objects.filter(election=election)
header = ('Centre #',
'Centre Name',
'Centre Type',
'Office #',
'Constituency #',
'Constituency Name',
'SubConstituency #',
'SubConstituency Name',
'Station number',
'Station Gender',
'Number of Registrants',
'First Name',
'First Name Number',
'Last Name',
'Last Name Number',
'When Generated',
)
rows = [header]
for station in stations:
center = station.center
rows.append((str(center.center_id),
center.name,
RegistrationCenter.Types.NAMES['ar'][center.center_type],
str(center.office.id),
str(center.constituency.id),
center.constituency.name_arabic,
str(center.subconstituency.id),
center.subconstituency.name_arabic,
str(station.number),
GENDER_NAMES[station.gender],
str(station.n_registrants),
station.first_voter_name,
str(station.first_voter_number),
station.last_voter_name,
str(station.last_voter_number),
station.creation_date.strftime('%Y-%m-%d %H:%M:%S')
))
faux_file = StringIO.StringIO()
csv = UnicodeWriter(faux_file)
csv.writerows(rows)
return faux_file.getvalue()
def download_blackwhitelisted_numbers(list_type):
"""Common handler for black- & whitelisted number download.
Permissions must be tested by the calling view; they're not checked here.
"""
# Technically this is not a CSV since there's only one column, but treating it as
# a CSV will make it easier to use with Excel, etc.
response = prepare_csv_response('{}list'.format(list_type))
writer = UnicodeWriter(response)
model = Blacklist if list_type == 'black' else Whitelist
for number in model.objects.all():
writer.writerow([number.phone_number])
return response
def download_blackwhitelisted_numbers(list_type):
"""Common handler for black- & whitelisted number download.
Permissions must be tested by the calling view; they're not checked here.
"""
# Technically this is not a CSV since there's only one column, but treating it as
# a CSV will make it easier to use with Excel, etc.
response = prepare_csv_response("{}list".format(list_type))
writer = UnicodeWriter(response)
model = Blacklist if list_type == "black" else Whitelist
for number in model.objects.all():
writer.writerow([number.phone_number])
return response
def generate_polling_metadata_csv():
election = Election.objects.get_most_current_election()
from rollgen.models import Station
stations = Station.objects.filter(election=election)
header = (
'Centre #',
'Centre Name',
'Centre Type',
'Office #',
'Constituency #',
'Constituency Name',
'SubConstituency #',
'SubConstituency Name',
'Station number',
'Station Gender',
'Number of Registrants',
'First Name',
'First Name Number',
'Last Name',
'Last Name Number',
'When Generated',
)
rows = [header]
for station in stations:
center = station.center
rows.append(
(str(center.center_id), center.name,
RegistrationCenter.Types.NAMES['ar'][center.center_type],
str(center.office.id), str(center.constituency.id),
center.constituency.name_arabic, str(center.subconstituency.id),
center.subconstituency.name_arabic,
str(station.number), GENDER_NAMES[station.gender],
str(station.n_registrants), station.first_voter_name,
str(station.first_voter_number), station.last_voter_name,
str(station.last_voter_number),
station.creation_date.strftime('%Y-%m-%d %H:%M:%S')))
faux_file = StringIO.StringIO()
csv = UnicodeWriter(faux_file)
csv.writerows(rows)
return faux_file.getvalue()
def download_centers_csv(request):
"""View to handle 'download centers as CSV' link"""
response = prepare_csv_response('centers')
writer = UnicodeWriter(response)
# write header row
writer.writerow(CSV_FIELDS)
for center in RegistrationCenter.objects.all():
writer.writerow([format_field(center, field) for field in CSV_FIELDS])
return response
def generate_rolls(self):
"""Build PDFs for this job. This is where all the action happens.
May raise NoVotersError, NoOfficeError and OutOfDiskSpaceError.
"""
self.begin = django_now()
if not self.input_arguments['forgive_no_office']:
# We are not going to be forgiving if we find any office-less centers.
has_office = lambda center: center.office.id != NO_NAMEDTHING
problem_centers = [center.center_id for center in self.centers if not
has_office(center)]
if problem_centers:
msg = "The following centers have no associated office: {}."
raise NoOfficeError(msg.format(problem_centers))
if not self.input_arguments['forgive_no_voters']:
# Test each center to make sure it has at least one registration. This is a lot of
# DB churn and can take a while. It has to be done in two parts.
# Find non-copy centers with no registrations
problem_center_ids = \
RegistrationCenter.objects.filter(id__in=[center.id for center in self.centers],
registration__isnull=True,
copy_of_id__isnull=True).values_list('id',
flat=True)
problem_centers = [center for center in self.centers if center.id in problem_center_ids]
# Find copy centers with no registrations. This runs one query per center which is
# the expensive way to do it, but it's the only way to figure out exactly which copy
# centers (if any) have parents with no registrations without dropping to raw SQL.
for center in self.centers:
copied = center.copy_of
if copied:
if not Registration.objects.filter(registration_center=copied).exists():
problem_centers.append(center)
if problem_centers:
problem_centers = [center.center_id for center in problem_centers]
msg = "The following centers have no registrants: {}."
raise NoVotersError(msg.format(problem_centers))
for i_center, center in enumerate(self.centers):
# Fetch the voters for this center from the DB.
voter_roll = get_voter_roll(center)
office_id = center.office.id
if office_id not in self.offices:
self.offices[office_id] = center.office
out_path = os.path.join(self.output_path, str(office_id))
if not os.path.exists(out_path):
with out_of_disk_space_handler_context():
os.makedirs(out_path)
filename_params = {'center_id': center.center_id, }
# Generate different PDFs based on phase
if self.phase == 'in-person':
# election center books only
for gender in (FEMALE, MALE):
filename_params['gender'] = GENDER_ABBRS[gender]
filename = self.get_filename(out_path, filename_params)
n_pages = generate_pdf(filename, center, voter_roll, gender, center_book=True)
self.add(filename, n_pages)
elif self.phase == 'exhibitions':
# election center list only
for gender in (FEMALE, MALE):
filename_params['gender'] = GENDER_ABBRS[gender]
filename = self.get_filename(out_path, filename_params)
n_pages = generate_pdf(filename, center, voter_roll, gender)
self.add(filename, n_pages)
elif self.phase == 'polling':
# distribute registrations into stations for this center
stations = station_distributor(voter_roll)
# Stash the list of which voters registered at this center/station for later.
election = Election.objects.get_most_current_election()
if not election:
raise NoElectionError('There is no current in-person election.')
for station in stations:
station.election = election
station.center = center
for voter in station.roll:
voter_station = VoterStation(national_id=voter.national_id,
center_id=center.center_id,
station_number=station.number)
self.voter_stations.append(voter_station)
# count stations by gender for center list
station_counts_by_gender = Counter(station.gender for station in stations)
for gender in station_counts_by_gender:
filename_params['gender'] = GENDER_ABBRS[gender]
filename = self.get_filename(out_path, filename_params, 'list')
n_pages = generate_pdf_center_list(filename, stations, gender)
self.add(filename, n_pages)
logger.info('center list {}'.format(filename))
# Create a separate book and sign for each station
for station in stations:
filename_params['station_number'] = station.number
# polling station books
filename = self.get_filename(out_path, filename_params, 'book')
n_pages = generate_pdf_station_book(filename, station)
self.add(filename, n_pages)
logger.info('station book {}'.format(filename))
# polling station sign
filename = self.get_filename(out_path, filename_params, 'sign')
n_pages = generate_pdf_station_sign(filename, station)
self.add(filename, n_pages)
logger.info('station book {}'.format(filename))
with transaction.atomic():
# Delete any existing Stations for this center and replace them with new.
Station.objects.filter(election=election, center=center).delete()
for station in stations:
station.save()
# Emit status
logger.info('saved PDFs for center %s' % center.center_id)
params = (i_center + 1, len(self.centers), (i_center + 1) / len(self.centers))
logger.info("Completed {} of {} (~{:.2%})".format(*params))
self.end = django_now()
# Now that rolls are generated, write voter station CSVs (if appropriate) and job JSON
# metadata. Last but not least, zip output.
if self.voter_stations:
# Write voter station data twice to CSV files. First sorted by national id and again
# sorted by (center id, station number).
header = [('national_id', 'center_id', 'station_number')]
# sort by national id
self.voter_stations.sort()
filename = os.path.join(self.output_path, 'voters_by_national_id.csv')
with out_of_disk_space_handler_context():
csv_writer = UnicodeWriter(open(filename, 'w'))
csv_writer.writerows(header)
csv_writer.writerows(self.voter_stations)
# sort by center, station number
self.voter_stations.sort(key=lambda voter_station: voter_station[1:])
filename = os.path.join(self.output_path, 'voters_by_center_and_station.csv')
with out_of_disk_space_handler_context():
csv_writer = UnicodeWriter(open(filename, 'w'))
csv_writer.writerows(header)
csv_writer.writerows(self.voter_stations)
# Write the JSON metadata file
metadata_filename = os.path.join(self.output_path, METADATA_FILENAME)
with out_of_disk_space_handler_context():
with open(metadata_filename, 'w') as f:
json.dump(self.metadata, f, indent=2)
# Write a hash of the metadata file
sha = hashlib.sha256(open(metadata_filename).read()).hexdigest()
with out_of_disk_space_handler_context():
open(metadata_filename + '.sha256', 'w').write(sha)
logger.info('zipping output')
for office_id in sorted(self.offices.keys()):
office_dir = os.path.join(self.output_path, str(office_id))
with out_of_disk_space_handler_context():
zip_filename = office_dir + '.zip'
with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as office_zip:
logger.info('zipping %s' % office_dir)
for office_base, dirs, files in os.walk(office_dir):
for pdf in files:
fn = os.path.join(office_base, pdf)
office_zip.write(fn, pdf)
logger.info('done')
