def generate_polling_metadata_csv(): election = Election.objects.get_most_current_election() from rollgen.models import Station stations = Station.objects.filter(election=election) header = ('Centre #', 'Centre Name', 'Centre Type', 'Office #', 'Constituency #', 'Constituency Name', 'SubConstituency #', 'SubConstituency Name', 'Station number', 'Station Gender', 'Number of Registrants', 'First Name', 'First Name Number', 'Last Name', 'Last Name Number', 'When Generated', ) rows = [header] for station in stations: center = station.center rows.append((str(center.center_id), center.name, RegistrationCenter.Types.NAMES['ar'][center.center_type], str(center.office.id), str(center.constituency.id), center.constituency.name_arabic, str(center.subconstituency.id), center.subconstituency.name_arabic, str(station.number), GENDER_NAMES[station.gender], str(station.n_registrants), station.first_voter_name, str(station.first_voter_number), station.last_voter_name, str(station.last_voter_number), station.creation_date.strftime('%Y-%m-%d %H:%M:%S') )) faux_file = StringIO.StringIO() csv = UnicodeWriter(faux_file) csv.writerows(rows) return faux_file.getvalue()
def generate_polling_metadata_csv(): election = Election.objects.get_most_current_election() from rollgen.models import Station stations = Station.objects.filter(election=election) header = ( 'Centre #', 'Centre Name', 'Centre Type', 'Office #', 'Constituency #', 'Constituency Name', 'SubConstituency #', 'SubConstituency Name', 'Station number', 'Station Gender', 'Number of Registrants', 'First Name', 'First Name Number', 'Last Name', 'Last Name Number', 'When Generated', ) rows = [header] for station in stations: center = station.center rows.append( (str(center.center_id), center.name, RegistrationCenter.Types.NAMES['ar'][center.center_type], str(center.office.id), str(center.constituency.id), center.constituency.name_arabic, str(center.subconstituency.id), center.subconstituency.name_arabic, str(station.number), GENDER_NAMES[station.gender], str(station.n_registrants), station.first_voter_name, str(station.first_voter_number), station.last_voter_name, str(station.last_voter_number), station.creation_date.strftime('%Y-%m-%d %H:%M:%S'))) faux_file = StringIO.StringIO() csv = UnicodeWriter(faux_file) csv.writerows(rows) return faux_file.getvalue()
def generate_rolls(self): """Build PDFs for this job. This is where all the action happens. May raise NoVotersError, NoOfficeError and OutOfDiskSpaceError. """ self.begin = django_now() if not self.input_arguments['forgive_no_office']: # We are not going to be forgiving if we find any office-less centers. has_office = lambda center: center.office.id != NO_NAMEDTHING problem_centers = [center.center_id for center in self.centers if not has_office(center)] if problem_centers: msg = "The following centers have no associated office: {}." raise NoOfficeError(msg.format(problem_centers)) if not self.input_arguments['forgive_no_voters']: # Test each center to make sure it has at least one registration. This is a lot of # DB churn and can take a while. It has to be done in two parts. # Find non-copy centers with no registrations problem_center_ids = \ RegistrationCenter.objects.filter(id__in=[center.id for center in self.centers], registration__isnull=True, copy_of_id__isnull=True).values_list('id', flat=True) problem_centers = [center for center in self.centers if center.id in problem_center_ids] # Find copy centers with no registrations. This runs one query per center which is # the expensive way to do it, but it's the only way to figure out exactly which copy # centers (if any) have parents with no registrations without dropping to raw SQL. for center in self.centers: copied = center.copy_of if copied: if not Registration.objects.filter(registration_center=copied).exists(): problem_centers.append(center) if problem_centers: problem_centers = [center.center_id for center in problem_centers] msg = "The following centers have no registrants: {}." raise NoVotersError(msg.format(problem_centers)) for i_center, center in enumerate(self.centers): # Fetch the voters for this center from the DB. voter_roll = get_voter_roll(center) office_id = center.office.id if office_id not in self.offices: self.offices[office_id] = center.office out_path = os.path.join(self.output_path, str(office_id)) if not os.path.exists(out_path): with out_of_disk_space_handler_context(): os.makedirs(out_path) filename_params = {'center_id': center.center_id, } # Generate different PDFs based on phase if self.phase == 'in-person': # election center books only for gender in (FEMALE, MALE): filename_params['gender'] = GENDER_ABBRS[gender] filename = self.get_filename(out_path, filename_params) n_pages = generate_pdf(filename, center, voter_roll, gender, center_book=True) self.add(filename, n_pages) elif self.phase == 'exhibitions': # election center list only for gender in (FEMALE, MALE): filename_params['gender'] = GENDER_ABBRS[gender] filename = self.get_filename(out_path, filename_params) n_pages = generate_pdf(filename, center, voter_roll, gender) self.add(filename, n_pages) elif self.phase == 'polling': # distribute registrations into stations for this center stations = station_distributor(voter_roll) # Stash the list of which voters registered at this center/station for later. election = Election.objects.get_most_current_election() if not election: raise NoElectionError('There is no current in-person election.') for station in stations: station.election = election station.center = center for voter in station.roll: voter_station = VoterStation(national_id=voter.national_id, center_id=center.center_id, station_number=station.number) self.voter_stations.append(voter_station) # count stations by gender for center list station_counts_by_gender = Counter(station.gender for station in stations) for gender in station_counts_by_gender: filename_params['gender'] = GENDER_ABBRS[gender] filename = self.get_filename(out_path, filename_params, 'list') n_pages = generate_pdf_center_list(filename, stations, gender) self.add(filename, n_pages) logger.info('center list {}'.format(filename)) # Create a separate book and sign for each station for station in stations: filename_params['station_number'] = station.number # polling station books filename = self.get_filename(out_path, filename_params, 'book') n_pages = generate_pdf_station_book(filename, station) self.add(filename, n_pages) logger.info('station book {}'.format(filename)) # polling station sign filename = self.get_filename(out_path, filename_params, 'sign') n_pages = generate_pdf_station_sign(filename, station) self.add(filename, n_pages) logger.info('station book {}'.format(filename)) with transaction.atomic(): # Delete any existing Stations for this center and replace them with new. Station.objects.filter(election=election, center=center).delete() for station in stations: station.save() # Emit status logger.info('saved PDFs for center %s' % center.center_id) params = (i_center + 1, len(self.centers), (i_center + 1) / len(self.centers)) logger.info("Completed {} of {} (~{:.2%})".format(*params)) self.end = django_now() # Now that rolls are generated, write voter station CSVs (if appropriate) and job JSON # metadata. Last but not least, zip output. if self.voter_stations: # Write voter station data twice to CSV files. First sorted by national id and again # sorted by (center id, station number). header = [('national_id', 'center_id', 'station_number')] # sort by national id self.voter_stations.sort() filename = os.path.join(self.output_path, 'voters_by_national_id.csv') with out_of_disk_space_handler_context(): csv_writer = UnicodeWriter(open(filename, 'w')) csv_writer.writerows(header) csv_writer.writerows(self.voter_stations) # sort by center, station number self.voter_stations.sort(key=lambda voter_station: voter_station[1:]) filename = os.path.join(self.output_path, 'voters_by_center_and_station.csv') with out_of_disk_space_handler_context(): csv_writer = UnicodeWriter(open(filename, 'w')) csv_writer.writerows(header) csv_writer.writerows(self.voter_stations) # Write the JSON metadata file metadata_filename = os.path.join(self.output_path, METADATA_FILENAME) with out_of_disk_space_handler_context(): with open(metadata_filename, 'w') as f: json.dump(self.metadata, f, indent=2) # Write a hash of the metadata file sha = hashlib.sha256(open(metadata_filename).read()).hexdigest() with out_of_disk_space_handler_context(): open(metadata_filename + '.sha256', 'w').write(sha) logger.info('zipping output') for office_id in sorted(self.offices.keys()): office_dir = os.path.join(self.output_path, str(office_id)) with out_of_disk_space_handler_context(): zip_filename = office_dir + '.zip' with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as office_zip: logger.info('zipping %s' % office_dir) for office_base, dirs, files in os.walk(office_dir): for pdf in files: fn = os.path.join(office_base, pdf) office_zip.write(fn, pdf) logger.info('done')