def copy_gemeente_resource(gemeente_code, source_resource, dest_resource,
dest_id=None, dest_hoofdstembureau=None,
dest_kieskring_id=None):
"""
Copies the records of a gemeente from one resource (source) to another
(dest). Note: this removes all records for the gemeente in dest first.
If dest contains no records then you need to specify the ID,
Hoofdstembureau and Kieskring ID value for the gemeente in the dest
resource.
"""
all_resource_records = ckan.get_records(source_resource)
gemeente_resource_records = [
record for record in all_resource_records['records']
if record['CBS gemeentecode'] == gemeente_code
]
_remove_id(gemeente_resource_records)
# If either one of these parameters is not set then try to get the
# values from the dest_resource
if not dest_id or not dest_hoofdstembureau or not dest_kieskring_id:
all_dest_resource_records = ckan.get_records(dest_resource)
gemeente_dest_resource_records = [
record for record in all_dest_resource_records['records']
if record['CBS gemeentecode'] == gemeente_code
]
if gemeente_dest_resource_records:
dest_id = gemeente_dest_resource_records[0]['ID']
dest_hoofdstembureau = gemeente_dest_resource_records[0][
'Hoofdstembureau'
]
dest_kieskring_id = gemeente_dest_resource_records[0][
'Kieskring ID'
]
# If either of these is still not set, abort!
if not dest_id or not dest_hoofdstembureau or not dest_kieskring_id:
print(
'Could not retrieve dest_id or dest_hoofdstembureau or '
'dest_kieskring_id'
)
for record in gemeente_resource_records:
record['ID'] = dest_id
record['Hoofdstembureau'] = dest_hoofdstembureau
record['Kieskring ID'] = dest_kieskring_id
ckan.delete_records(
dest_resource,
{'CBS gemeentecode': gemeente_code}
)
ckan.save_records(dest_resource, gemeente_resource_records)
def export_resource(resource_id):
"""
Exports all records of a resource to a json file in the exports directory
"""
all_resource_records = ckan.get_records(resource_id)['records']
filename = 'exports/%s_%s.json' % (datetime.now().isoformat()[:19],
resource_id)
with open(filename, 'w') as OUT:
json.dump(all_resource_records, OUT, indent=4, sort_keys=True)
def fix_bag_addresses(resource_type):
"""
Checks all records of all election resources (default draft) for
missing address information. If this is the case, try to retrieve
it based on the available BAG number, otherwise check if the BAG
number is another type of BAG number (other then nummeraanduiding)
and see if it corresponds with 1 BAG nummeraanduiding ID in order
to retrieve the address. Finally, the whole resource is exported.
"""
for name, election in ckan.elections.items():
total = 0
bag_found = 0
resource_id = election['%s_resource' % (resource_type)]
sys.stderr.write('%s: %s\n' % (name, resource_id))
records = ckan.get_records(resource_id)
for r in records['records']:
bag = _get_bag(r)
if bag:
bag_found += 1
r['BAG referentienummer'] = bag.nummeraanduiding
bag_conversions = {
'verblijfsobjectgebruiksdoel': 'Gebruikersdoel het gebouw',
'openbareruimte': 'Straatnaam',
'huisnummer': 'Huisnummer',
'huisletter': 'Huisletter',
'huisnummertoevoeging': 'Huisnummertoevoeging',
'postcode': 'Postcode',
'woonplaats': 'Plaats'
}
for bag_field, record_field in bag_conversions.items():
bag_field_value = getattr(bag, bag_field, None)
if bag_field_value is not None:
r[record_field] = bag_field_value.encode(
'latin1'
).decode()
else:
r[record_field] = None
total += 1
sys.stderr.write(
"%s records, %s with BAG found\n" % (
total, bag_found
)
)
with open('exports/%s_bag_add_fix.json' % (resource_id), 'w') as OUT:
json.dump(records['records'], OUT, indent=4, sort_keys=True)
def publish_gemeente(gemeente_code):
"""
Publishes the saved (draft) stembureaus of a gemeente
"""
current_gemeente = _get_gemeente(gemeente_code)
elections = current_gemeente.elections.all()
for election in [x.verkiezing for x in elections]:
temp_all_draft_records = ckan.get_records(
ckan.elections[election]['draft_resource']
)
temp_gemeente_draft_records = [
record for record in temp_all_draft_records['records']
if record['CBS gemeentecode'] == current_gemeente.gemeente_code
]
_remove_id(temp_gemeente_draft_records)
ckan.publish(election, current_gemeente.gemeente_code, temp_gemeente_draft_records)
def upload_stembureau_spreadsheet(gemeente_code, file_path):
"""
Uploads a stembureau spreadheet, specify full absolute file_path
"""
current_gemeente = _get_gemeente(gemeente_code)
elections = current_gemeente.elections.all()
# Pick the first election. In the case of multiple elections we only
# retrieve the stembureaus of the first election as the records for
# both elections are the same (at least the GR2018 + referendum
# elections on March 21st 2018).
verkiezing = elections[0].verkiezing
all_draft_records = ckan.get_records(
ckan.elections[verkiezing]['draft_resource']
)
gemeente_draft_records = [
record for record in all_draft_records['records']
if record['CBS gemeentecode'] == current_gemeente.gemeente_code
]
_remove_id(gemeente_draft_records)
parser = UploadFileParser()
app.logger.info(
'Manually (CLI) uploading file for '
'%s' % (current_gemeente.gemeente_naam)
)
try:
records = parser.parse(file_path)
except ValueError as e:
app.logger.warning('Manual upload failed: %s' % e)
return
validator = Validator()
results = validator.validate(records)
# If the spreadsheet did not validate then return the errors
if not results['no_errors']:
print(
'Upload failed. Fix the errors shown below and try again.\n\n'
)
for column_number, col_result in sorted(
results['results'].items()):
if col_result['errors']:
print(
'Error(s) in '
'invulveld %s:' % (
column_number - 5
)
)
for column_name, error in col_result['errors'].items():
print(
'%s: %s\n' % (
column_name, error[0]
)
)
# If there is not a single value in the results then state that we
# could not find any stembureaus
elif not results['found_any_record_with_values']:
print(
'Upload failed. No stembureaus have been found in this '
'spreadsheet.'
)
# If the spreadsheet did validate then first delete all current
# stembureaus from the draft_resource and then save the newly
# uploaded stembureaus to the draft_resources of each election
else:
# Delete all stembureaus of current gemeente
if gemeente_draft_records:
for election in [x.verkiezing for x in elections]:
ckan.delete_records(
ckan.elections[election]['draft_resource'],
{
'CBS gemeentecode': current_gemeente.gemeente_code
}
)
# Create and save records
for election in [x.verkiezing for x in elections]:
records = []
for _, result in results['results'].items():
if result['form']:
records.append(
_create_record(
result['form'],
result['uuid'],
current_gemeente,
election
)
)
ckan.save_records(
ckan.elections[election]['draft_resource'],
records=records
)
print('Upload succesful!')
print('\n\n')
def gemeente_stemlokalen_edit(stemlokaal_id=None):
# Select a gemeente if none is currently selected
if not session['selected_gemeente_code']:
return redirect(url_for('gemeente_selectie'))
gemeente = Gemeente.query.filter_by(
gemeente_code=session['selected_gemeente_code']).first()
elections = gemeente.elections.all()
# Pick the first election. In the case of multiple elections we only
# retrieve the stembureaus of the first election as the records for
# both elections are the same (at least the GR2018 + referendum
# elections on March 21st 2018).
verkiezing = elections[0].verkiezing
all_draft_records = ckan.get_records(
ckan.elections[verkiezing]['draft_resource'])
gemeente_draft_records = [
record for record in all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
# Initialize the form with the data already available in the draft
init_record = {}
if stemlokaal_id:
for record in gemeente_draft_records:
if record['UUID'] == stemlokaal_id:
# Split the Verkiezingen attribute into a list
if record['Verkiezingen']:
record['Verkiezingen'] = [
x.strip() for x in record['Verkiezingen'].split(';')
]
init_record = Record(
**{k.lower(): v
for k, v in record.items()}).record
form = EditForm(**init_record)
# When the user clicked the 'Annuleren' button go back to the
# overzicht page without doing anything
if form.submit_annuleren.data:
flash('Bewerking geannuleerd')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
# When the user clicked the 'Verwijderen' button delete the
# stembureau from the draft_resources of each election
if form.submit_verwijderen.data:
if stemlokaal_id:
for election in [x.verkiezing for x in elections]:
ckan.delete_records(ckan.elections[election]['draft_resource'],
{'UUID': stemlokaal_id})
flash('Stembureau verwijderd')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
# When the user clicked the 'Opslaan' button save the stembureau
# to the draft_resources of each election
if form.validate_on_submit():
if not stemlokaal_id:
stemlokaal_id = uuid.uuid4().hex
for election in [x.verkiezing for x in elections]:
record = _create_record(form, stemlokaal_id, gemeente, election)
ckan.save_records(ckan.elections[election]['draft_resource'],
records=[record])
flash('Stembureau opgeslagen')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
return render_template('gemeente-stemlokalen-edit.html',
form=form,
gemeente=gemeente,
upload_deadline_passed=check_deadline_passed())
def gemeente_stemlokalen_overzicht():
# Select a gemeente if none is currently selected
if not session['selected_gemeente_code']:
return redirect(url_for('gemeente_selectie'))
gemeente = Gemeente.query.filter_by(
gemeente_code=session['selected_gemeente_code']).first()
elections = gemeente.elections.all()
# Pick the first election. In the case of multiple elections we only
# retrieve the stembureaus of the first election as the records for
# both elections are the same (at least the GR2018 + referendum
# elections on March 21st 2018).
verkiezing = elections[0].verkiezing
all_draft_records = ckan.get_records(
ckan.elections[verkiezing]['draft_resource'])
gemeente_draft_records = [
record for record in all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(gemeente_draft_records)
publish_form = PubliceerForm()
# Publiceren
if publish_form.validate_on_submit():
if publish_form.submit.data:
# Publish stembureaus to all elections
for election in [x.verkiezing for x in elections]:
temp_all_draft_records = ckan.get_records(
ckan.elections[election]['draft_resource'])
temp_gemeente_draft_records = [
record for record in temp_all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(temp_gemeente_draft_records)
ckan.publish(election, gemeente.gemeente_code,
temp_gemeente_draft_records)
flash('Stembureaus gepubliceerd')
# Sleep to make sure that the data is saved before it is
# requested again in the lines right below here
sleep(1)
all_publish_records = ckan.get_records(
ckan.elections[verkiezing]['publish_resource'])
gemeente_publish_records = [
record for record in all_publish_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(gemeente_publish_records)
# Check whether gemeente_draft_records differs from
# gemeente_publish_records in order to disable or enable the 'Publiceer'
# button
disable_publish_form = True
if gemeente_draft_records != gemeente_publish_records:
disable_publish_form = False
# Pagination
posts_per_page = app.config['POSTS_PER_PAGE']
page = request.args.get('page', 1, type=int)
# Use page 1 if a page lower than 1 is requested
if page < 1:
page = 1
# If the user requests a page larger than the largest page for which
# we have records to show, use that page instead of the requested
# one
if page > ceil(len(gemeente_draft_records) / posts_per_page) and page > 1:
page = ceil(len(gemeente_draft_records) / posts_per_page)
start_record = (page - 1) * posts_per_page
end_record = page * posts_per_page
if end_record > len(gemeente_draft_records):
end_record = len(gemeente_draft_records)
sorted_draft_records = sorted(gemeente_draft_records,
key=lambda k: k['Stembureau of Afgiftepunt'])
paged_draft_records = sorted_draft_records[start_record:end_record]
previous_url = None
if page > 1:
previous_url = url_for('gemeente_stemlokalen_overzicht', page=page - 1)
next_url = None
if len(gemeente_draft_records) > page * posts_per_page:
next_url = url_for('gemeente_stemlokalen_overzicht', page=page + 1)
# Only add 1 if there are records
if len(gemeente_draft_records):
start_record += 1
total_pages = ceil(len(gemeente_draft_records) / posts_per_page)
if total_pages == 0:
total_pages = 1
return render_template(
'gemeente-stemlokalen-overzicht.html',
verkiezing_string=_format_verkiezingen_string(elections),
gemeente=gemeente,
draft_records=paged_draft_records,
field_order=field_order,
publish_form=publish_form,
disable_publish_form=disable_publish_form,
page=page,
start_record=start_record,
end_record=end_record,
total_records=len(gemeente_draft_records),
total_pages=total_pages,
previous_url=previous_url,
next_url=next_url,
upload_deadline_passed=check_deadline_passed())
def gemeente_stemlokalen_dashboard():
# Select a gemeente if none is currently selected
if not 'selected_gemeente_code' in session:
return redirect(url_for('gemeente_selectie'))
gemeente = Gemeente.query.filter_by(
gemeente_code=session['selected_gemeente_code']).first()
elections = gemeente.elections.all()
# Pick the first election. In the case of multiple elections we only
# retrieve the stembureaus of the first election as the records for
# both elections are the same (at least the GR2018 + referendum
# elections on March 21st 2018).
verkiezing = elections[0].verkiezing
all_publish_records = ckan.get_records(
ckan.elections[verkiezing]['publish_resource'])
all_draft_records = ckan.get_records(
ckan.elections[verkiezing]['draft_resource'])
gemeente_publish_records = [
record for record in all_publish_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
gemeente_draft_records = [
record for record in all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(gemeente_publish_records)
_remove_id(gemeente_draft_records)
toon_stembureaus_pagina = False
if gemeente_publish_records:
toon_stembureaus_pagina = True
show_publish_note = False
if gemeente_draft_records != gemeente_publish_records:
show_publish_note = True
vooringevuld = ''
vooringevuld_fn = (
'files/deels_vooringevuld/waarismijnstemlokaal.nl_invulformulier_%s_'
'deels_vooringevuld.xlsx' % (gemeente.gemeente_naam))
if os.path.exists(vooringevuld_fn):
vooringevuld = vooringevuld_fn
form = FileUploadForm()
# Save, parse and validate an uploaded spreadsheet and save the
# stembureaus
if form.validate_on_submit():
f = form.data_file.data
filename = secure_filename(f.filename)
filename = '%s__%s' % (gemeente.gemeente_code, filename)
file_path = os.path.join(
os.path.abspath(os.path.join(app.instance_path, '../upload')),
filename)
f.save(file_path)
parser = UploadFileParser()
app.logger.info('Processing uploaded file for %s' %
(gemeente.gemeente_naam))
try:
records = parser.parse(file_path)
except ValueError as e:
app.logger.warning('Upload failed: %s' % e)
flash(
Markup(
'<span class="text-red">Uploaden mislukt</span>. Het '
'lijkt er op dat u geen gebruik maakt van (de meest '
'recente versie van) de stembureau-spreadsheet. Download '
'een <a href="/files/waarismijnstemlokaal.nl_'
'invulformulier.xlsx"><b>leeg</b></a> of <a href="%s"><b>'
'deels vooringevuld</b></a> stembureau-spreadsheet en vul '
'de gegevens volgens de instructies in de spreadsheet in '
'om deze vervolgens op deze pagina te '
'uploaden.' % (vooringevuld)))
return render_template(
'gemeente-stemlokalen-dashboard.html',
verkiezing_string=_format_verkiezingen_string(elections),
gemeente=gemeente,
total_publish_records=len(gemeente_publish_records),
total_draft_records=len(gemeente_draft_records),
form=form,
show_publish_note=show_publish_note,
vooringevuld=vooringevuld,
toon_stembureaus_pagina=toon_stembureaus_pagina,
upload_deadline_passed=check_deadline_passed())
validator = Validator()
results = validator.validate(records)
# If the spreadsheet did not validate then return the errors as
# flash messages
if not results['no_errors']:
flash(
Markup(
'<span class="text-red">Uploaden mislukt</span>. Los de '
'hieronder getoonde foutmeldingen op en upload de '
'spreadsheet opnieuw.'
'<br><br>'))
for column_number, col_result in sorted(
results['results'].items()):
if col_result['errors']:
error_flash = (
'<b>Foutmelding(en) in <span class="text-red">'
'invulveld %s (oftewel kolom "%s")</span></b>:' %
(column_number - 5, _colnum2string(column_number)))
error_flash += '<ul>'
for column_name, error in col_result['errors'].items():
error_flash += '<li>%s: %s</li>' % (column_name,
error[0])
error_flash += '</ul><br>'
flash(Markup(error_flash))
# If there not a single value in the results then state that we
# could not find any stembureaus
elif not results['found_any_record_with_values']:
flash(
Markup(
'<span class="text-red">Uploaden mislukt</span>. Er zijn geen '
'stembureaus gevonden in de spreadsheet.'))
# If the spreadsheet did validate then first delete all current
# stembureaus from the draft_resource and then save the newly
# uploaded stembureaus to the draft_resources of each election
# and finally redirect to the overzicht
else:
# Delete all stembureaus of current gemeente
if gemeente_draft_records:
for election in [x.verkiezing for x in elections]:
ckan.delete_records(
ckan.elections[election]['draft_resource'],
{'CBS gemeentecode': gemeente.gemeente_code})
# Create and save records
for election in [x.verkiezing for x in elections]:
records = []
for _, result in results['results'].items():
if result['form']:
records.append(
_create_record(result['form'], result['uuid'],
gemeente, election))
ckan.save_records(ckan.elections[election]['draft_resource'],
records=records)
flash(
'Het uploaden van stembureaus is gelukt! Controleer in het '
'overzicht hieronder of alles klopt en voer eventuele '
'wijzigingen door. Klik vervolgens op de "Publiceer"-knop als '
'alles klopt.')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
return render_template(
'gemeente-stemlokalen-dashboard.html',
verkiezing_string=_format_verkiezingen_string(elections),
gemeente=gemeente,
total_publish_records=len(gemeente_publish_records),
total_draft_records=len(gemeente_draft_records),
form=form,
show_publish_note=show_publish_note,
vooringevuld=vooringevuld,
toon_stembureaus_pagina=toon_stembureaus_pagina,
upload_deadline_passed=check_deadline_passed())
def gemeente_stemlokalen_edit(stemlokaal_id=None):
# Select a gemeente if none is currently selected
if not 'selected_gemeente_code' in session:
return redirect(url_for('gemeente_selectie'))
gemeente = Gemeente.query.filter_by(
gemeente_code=session['selected_gemeente_code']).first()
elections = gemeente.elections.all()
# Need this to get a starting point for the clickmap;
# Uses re.sub to remove provinces from some gemeenten which is how we write
# gemeenten in WIMS, but which are not used in the BAG, e.g. 'Beek (L.)',
# but keep 'Bergen (NH.)' and 'Bergen (L.)' as the BAG also uses that
# spelling.
# TODO this won't work for BES-eilanden as they don't exist in the BAG, so
# exclude them from the filter below and initialize a custom bag_record
# with coordinates for the BES-eilanden.
bag_record = BAG.query.filter_by(
gemeente=gemeente.gemeente_naam if 'Bergen (' in gemeente.gemeente_naam
else re.sub(' \(.*\)$', '', gemeente.gemeente_naam)).order_by(
'openbareruimte').first()
# Pick the first election. In the case of multiple elections we only
# retrieve the stembureaus of the first election as the records for
# both elections are the same (at least for the GR2018 + referendum
# elections on March 21st 2018).
verkiezing = elections[0].verkiezing
all_draft_records = ckan.get_records(
ckan.elections[verkiezing]['draft_resource'])
gemeente_draft_records = [
record for record in all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
# Initialize the form with the data already available in the draft
init_record = {}
if stemlokaal_id:
for record in gemeente_draft_records:
if record['UUID'] == stemlokaal_id:
# Split the Verkiezingen attribute into a list
if record['Verkiezingen']:
record['Verkiezingen'] = [
x.strip() for x in record['Verkiezingen'].split(';')
]
init_record = Record(
**{k.lower(): v
for k, v in record.items()}).record
form = EditForm(**init_record)
# When the user clicked the 'Annuleren' button go back to the
# overzicht page without doing anything
if form.submit_annuleren.data:
flash('Bewerking geannuleerd')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
# When the user clicked the 'Verwijderen' button delete the
# stembureau from the draft_resources of each election
if form.submit_verwijderen.data:
if stemlokaal_id:
for election in [x.verkiezing for x in elections]:
ckan.delete_records(ckan.elections[election]['draft_resource'],
{'UUID': stemlokaal_id})
flash('Stembureau verwijderd')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
# When the user clicked the 'Opslaan' button save the stembureau
# to the draft_resources of each election
if form.validate_on_submit():
if not stemlokaal_id:
stemlokaal_id = uuid.uuid4().hex
for election in [x.verkiezing for x in elections]:
record = _create_record(form, stemlokaal_id, gemeente, election)
ckan.save_records(ckan.elections[election]['draft_resource'],
records=[record])
flash('Stembureau opgeslagen')
return redirect(url_for('gemeente_stemlokalen_overzicht'))
return render_template('gemeente-stemlokalen-edit.html',
form=form,
gemeente=gemeente,
bag_record=bag_record,
upload_deadline_passed=check_deadline_passed())
def gemeente_stemlokalen_overzicht():
# Select a gemeente if none is currently selected
if not 'selected_gemeente_code' in session:
return redirect(url_for('gemeente_selectie'))
gemeente = Gemeente.query.filter_by(
gemeente_code=session['selected_gemeente_code']).first()
elections = gemeente.elections.all()
# Pick the first election. In the case of multiple elections we only
# retrieve the stembureaus of the first election as the records for
# both elections are the same (at least for the GR2018 + referendum
# elections on March 21st 2018).
verkiezing = elections[0].verkiezing
all_draft_records = ckan.get_records(
ckan.elections[verkiezing]['draft_resource'])
gemeente_draft_records = [
record for record in all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(gemeente_draft_records)
publish_form = PubliceerForm()
# Publiceren
if publish_form.validate_on_submit():
if publish_form.submit.data:
# Publish stembureaus to all elections
for election in [x.verkiezing for x in elections]:
temp_all_draft_records = ckan.get_records(
ckan.elections[election]['draft_resource'])
temp_gemeente_draft_records = [
record for record in temp_all_draft_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(temp_gemeente_draft_records)
ckan.publish(election, gemeente.gemeente_code,
temp_gemeente_draft_records)
flash('Stembureaus gepubliceerd')
# Sleep to make sure that the data is saved before it is
# requested again in the lines right below here
sleep(1)
all_publish_records = ckan.get_records(
ckan.elections[verkiezing]['publish_resource'])
gemeente_publish_records = [
record for record in all_publish_records['records']
if record['CBS gemeentecode'] == gemeente.gemeente_code
]
_remove_id(gemeente_publish_records)
# Check whether gemeente_draft_records differs from
# gemeente_publish_records in order to disable or enable the 'Publiceer'
# button
disable_publish_form = True
if gemeente_draft_records != gemeente_publish_records:
disable_publish_form = False
return render_template(
'gemeente-stemlokalen-overzicht.html',
verkiezing_string=_format_verkiezingen_string(elections),
gemeente=gemeente,
draft_records=gemeente_draft_records,
field_order=field_order,
publish_form=publish_form,
disable_publish_form=disable_publish_form,
upload_deadline_passed=check_deadline_passed())
