def test_datastore_upsert(resource_id): """ Insert or update data in the datastore table in a resource with an example record; used for testing """ record = { "Gemeente": "'s-Gravenhage", "CBS gemeentecode": "GM0518", "Stembureau of Afgiftepunt": "Stembureau", "Nummer stembureau": "517", "Naam stembureau": "Stadhuis", "Gebruikersdoel het gebouw": "kantoor", "Website locatie": ( "https://www.denhaag.nl/nl/bestuur-en-organisatie/contact-met-" "de-gemeente/stadhuis-den-haag.htm" ), "Wijknaam": "Centrum", "CBS wijknummer": "WK051828", "Buurtnaam": "Kortenbos", "CBS buurtnummer": "BU05182811", "BAG referentienummer": "0518100000275247", "Straatnaam": "Spui", "Huisnummer": 70, "Huisletter": "", "Huisnummertoevoeging": "", "Postcode": "2511 BT", "Plaats": "Den Haag", "Extra adresaanduiding": "", "X": 81611, "Y": 454909, "Longitude": 4.3166395, "Latitude": 52.0775912, "Openingstijden 10-03-2021": "2021-03-10T07:30:00 tot 2021-03-10T21:00:00", "Openingstijden 11-03-2021": "2021-03-11T07:30:00 tot 2021-03-11T21:00:00", "Openingstijden 12-03-2021": "2021-03-12T07:30:00 tot 2021-03-12T21:00:00", "Openingstijden 13-03-2021": "2021-03-13T07:30:00 tot 2021-03-13T21:00:00", "Openingstijden 14-03-2021": "2021-03-14T07:30:00 tot 2021-03-14T21:00:00", "Openingstijden 15-03-2021": "2021-03-15T07:30:00 tot 2021-03-15T21:00:00", "Openingstijden 16-03-2021": "2021-03-16T07:30:00 tot 2021-03-16T21:00:00", "Openingstijden 17-03-2021": "2021-03-17T07:30:00 tot 2021-03-17T21:00:00", "Mindervaliden toegankelijk": "Y", "Akoestiek": "Y", "Auditieve hulpmiddelen": "Doventolk, ringleiding", "Visuele hulpmiddelen": "Leesloep", "Mindervalide toilet aanwezig": 'N', "Kieskring ID": "'s-Gravenhage", "Hoofdstembureau": "Nederland", "Tellocatie": "Y", "Contactgegevens": "*****@*****.**", "Beschikbaarheid": "https://www.stembureausindenhaag.nl/", "Verkiezingen": "", "ID": "NLODSGM0518stembureaus20180321001", "UUID": uuid.uuid4().hex } ckan.save_records( resource_id=resource_id, records=[record] )
def import_resource(resource_id, file_path): """ Import records to a resource from a json file """ with open(file_path) as IN: records = json.load(IN) for record in records: if '_id' in record: del record['_id'] ckan.save_records( resource_id, sorted(records, key=lambda x: (x['Gemeente'], x['Nummer stembureau'])))
def copy_gemeente_resource(gemeente_code, source_resource, dest_resource, dest_id=None, dest_hoofdstembureau=None, dest_kieskring_id=None): """ Copies the records of a gemeente from one resource (source) to another (dest). Note: this removes all records for the gemeente in dest first. If dest contains no records then you need to specify the ID, Hoofdstembureau and Kieskring ID value for the gemeente in the dest resource. """ all_resource_records = ckan.get_records(source_resource) gemeente_resource_records = [ record for record in all_resource_records['records'] if record['CBS gemeentecode'] == gemeente_code ] _remove_id(gemeente_resource_records) # If either one of these parameters is not set then try to get the # values from the dest_resource if not dest_id or not dest_hoofdstembureau or not dest_kieskring_id: all_dest_resource_records = ckan.get_records(dest_resource) gemeente_dest_resource_records = [ record for record in all_dest_resource_records['records'] if record['CBS gemeentecode'] == gemeente_code ] if gemeente_dest_resource_records: dest_id = gemeente_dest_resource_records[0]['ID'] dest_hoofdstembureau = gemeente_dest_resource_records[0][ 'Hoofdstembureau' ] dest_kieskring_id = gemeente_dest_resource_records[0][ 'Kieskring ID' ] # If either of these is still not set, abort! if not dest_id or not dest_hoofdstembureau or not dest_kieskring_id: print( 'Could not retrieve dest_id or dest_hoofdstembureau or ' 'dest_kieskring_id' ) for record in gemeente_resource_records: record['ID'] = dest_id record['Hoofdstembureau'] = dest_hoofdstembureau record['Kieskring ID'] = dest_kieskring_id ckan.delete_records( dest_resource, {'CBS gemeentecode': gemeente_code} ) ckan.save_records(dest_resource, gemeente_resource_records)
def import_rug(rug_file_path, excluded_gemeenten_file_path, gemeenten_info_file_path): """ Import records coming from Geodienst from the Rijksuniversiteit Groningen. These records don't contain all fields and these need to be filled. Based on the gemeente in the record it will be saved to correct election(s) resources (draft + publish). """ # Retrieve information about gemeenten with open(gemeenten_info_file_path) as IN: gemeenten_info = json.load(IN) # Retrieve file containing a list of names of gemeenten which # uploaded stembureaus themselves and thus don't need to be # retrieved from the RUG data with open(excluded_gemeenten_file_path) as IN: excluded_gemeenten = [line.strip() for line in IN] with open(rug_file_path) as IN: # Load RUG file rug_records = json.load(IN) resource_records = {} # Prepopulate a dict with all CKAN resources for election, values in app.config['CKAN_CURRENT_ELECTIONS'].items(): resource_records[values['draft_resource']] = [] resource_records[values['publish_resource']] = [] # Process each record for rug_record in rug_records: # Skip record if its gemeente is in the excluded list if rug_record['Gemeente'] in excluded_gemeenten: continue # Retrieve the gemeente info for the gemeente of the # current record record_gemeente_info = {} for gemeente_info in gemeenten_info: if gemeente_info['gemeente_naam'] == rug_record['Gemeente']: record_gemeente_info = gemeente_info rug_record['UUID'] = uuid.uuid4().hex gemeente_code = record_gemeente_info['gemeente_code'] rug_record['CBS gemeentecode'] = gemeente_code # Try to retrieve the record in the BAG bag_result = BAG.query.filter_by( openbareruimte=rug_record['Straatnaam'], huisnummer=rug_record['Huisnummer'], huisnummertoevoeging=rug_record['Huisnummertoevoeging'], woonplaats=rug_record['Plaats'] ) # If the query above didn't work, try it again without # huisnummertoevoeging if bag_result.count() == 0: bag_result = BAG.query.filter_by( openbareruimte=rug_record['Straatnaam'], huisnummer=rug_record['Huisnummer'], woonplaats=rug_record['Plaats'] ) # If there are multiple BAG matches, simply take the first bag_object = bag_result.first() # Retrieve gebruikersdoel, postcode and nummeraanduiding # from BAG if bag_object: bag_conversions = { 'verblijfsobjectgebruiksdoel': 'Gebruikersdoel het gebouw', 'postcode': 'Postcode', 'nummeraanduiding': 'BAG referentienummer' } for bag_field, record_field in bag_conversions.items(): bag_field_value = getattr(bag_object, bag_field, None) if bag_field_value is not None: rug_record[record_field] = bag_field_value.encode( 'latin1' ).decode() else: rug_record[record_field] = None ## We stopped adding the wijk and buurt data as the data ## supplied by CBS is not up to date enough as it is only ## released once a year and many months after changes ## have been made by the municipalities. # Retrieve wijk and buurt info #wk_code, wk_naam, bu_code, bu_naam = find_buurt_and_wijk( # '000', # rug_record['CBS gemeentecode'], # rug_record['Longitude'], # rug_record['Latitude'] #) #if wk_naam: # rug_record['Wijknaam'] = wk_naam #if wk_code: # rug_record['CBS wijknummer'] = wk_code #if bu_naam: # rug_record['Buurtnaam'] = bu_naam #if bu_code: # rug_record['CBS buurtnummer'] = bu_code # Loop over each election in which the current gemeente # participates and create election specific fields for verkiezing in record_gemeente_info['verkiezingen']: record = copy.deepcopy(rug_record) verkiezing_info = app.config['CKAN_CURRENT_ELECTIONS'][ verkiezing ] record['ID'] = 'NLODS%sstembureaus%s%s' % ( gemeente_code, verkiezing_info['election_date'], verkiezing_info['election_number'] ) kieskring_id = '' hoofdstembureau = '' if verkiezing.startswith('Gemeenteraadsverkiezingen'): kieskring_id = record['Gemeente'] hoofdstembureau = record['Gemeente'] if verkiezing.startswith('Referendum'): for row in kieskringen: if row[2] == record['Gemeente']: kieskring_id = row[0] hoofdstembureau = row[1] record['Kieskring ID'] = kieskring_id record['Hoofdstembureau'] = hoofdstembureau # Append the record for the draft and publish resource # of this election resources = [ verkiezing_info['draft_resource'], verkiezing_info['publish_resource'] ] for resource in resources: resource_records[resource].append(record) for resource, res_records in resource_records.items(): print('%s: %s' % (resource, len(res_records))) ckan.save_records(resource, res_records)
def upload_stembureau_spreadsheet(gemeente_code, file_path): """ Uploads a stembureau spreadheet, specify full absolute file_path """ current_gemeente = _get_gemeente(gemeente_code) elections = current_gemeente.elections.all() # Pick the first election. In the case of multiple elections we only # retrieve the stembureaus of the first election as the records for # both elections are the same (at least the GR2018 + referendum # elections on March 21st 2018). verkiezing = elections[0].verkiezing all_draft_records = ckan.get_records( ckan.elections[verkiezing]['draft_resource'] ) gemeente_draft_records = [ record for record in all_draft_records['records'] if record['CBS gemeentecode'] == current_gemeente.gemeente_code ] _remove_id(gemeente_draft_records) parser = UploadFileParser() app.logger.info( 'Manually (CLI) uploading file for ' '%s' % (current_gemeente.gemeente_naam) ) try: records = parser.parse(file_path) except ValueError as e: app.logger.warning('Manual upload failed: %s' % e) return validator = Validator() results = validator.validate(records) # If the spreadsheet did not validate then return the errors if not results['no_errors']: print( 'Upload failed. Fix the errors shown below and try again.\n\n' ) for column_number, col_result in sorted( results['results'].items()): if col_result['errors']: print( 'Error(s) in ' 'invulveld %s:' % ( column_number - 5 ) ) for column_name, error in col_result['errors'].items(): print( '%s: %s\n' % ( column_name, error[0] ) ) # If there is not a single value in the results then state that we # could not find any stembureaus elif not results['found_any_record_with_values']: print( 'Upload failed. No stembureaus have been found in this ' 'spreadsheet.' ) # If the spreadsheet did validate then first delete all current # stembureaus from the draft_resource and then save the newly # uploaded stembureaus to the draft_resources of each election else: # Delete all stembureaus of current gemeente if gemeente_draft_records: for election in [x.verkiezing for x in elections]: ckan.delete_records( ckan.elections[election]['draft_resource'], { 'CBS gemeentecode': current_gemeente.gemeente_code } ) # Create and save records for election in [x.verkiezing for x in elections]: records = [] for _, result in results['results'].items(): if result['form']: records.append( _create_record( result['form'], result['uuid'], current_gemeente, election ) ) ckan.save_records( ckan.elections[election]['draft_resource'], records=records ) print('Upload succesful!') print('\n\n')
def gemeente_stemlokalen_edit(stemlokaal_id=None): # Select a gemeente if none is currently selected if not session['selected_gemeente_code']: return redirect(url_for('gemeente_selectie')) gemeente = Gemeente.query.filter_by( gemeente_code=session['selected_gemeente_code']).first() elections = gemeente.elections.all() # Pick the first election. In the case of multiple elections we only # retrieve the stembureaus of the first election as the records for # both elections are the same (at least the GR2018 + referendum # elections on March 21st 2018). verkiezing = elections[0].verkiezing all_draft_records = ckan.get_records( ckan.elections[verkiezing]['draft_resource']) gemeente_draft_records = [ record for record in all_draft_records['records'] if record['CBS gemeentecode'] == gemeente.gemeente_code ] # Initialize the form with the data already available in the draft init_record = {} if stemlokaal_id: for record in gemeente_draft_records: if record['UUID'] == stemlokaal_id: # Split the Verkiezingen attribute into a list if record['Verkiezingen']: record['Verkiezingen'] = [ x.strip() for x in record['Verkiezingen'].split(';') ] init_record = Record( **{k.lower(): v for k, v in record.items()}).record form = EditForm(**init_record) # When the user clicked the 'Annuleren' button go back to the # overzicht page without doing anything if form.submit_annuleren.data: flash('Bewerking geannuleerd') return redirect(url_for('gemeente_stemlokalen_overzicht')) # When the user clicked the 'Verwijderen' button delete the # stembureau from the draft_resources of each election if form.submit_verwijderen.data: if stemlokaal_id: for election in [x.verkiezing for x in elections]: ckan.delete_records(ckan.elections[election]['draft_resource'], {'UUID': stemlokaal_id}) flash('Stembureau verwijderd') return redirect(url_for('gemeente_stemlokalen_overzicht')) # When the user clicked the 'Opslaan' button save the stembureau # to the draft_resources of each election if form.validate_on_submit(): if not stemlokaal_id: stemlokaal_id = uuid.uuid4().hex for election in [x.verkiezing for x in elections]: record = _create_record(form, stemlokaal_id, gemeente, election) ckan.save_records(ckan.elections[election]['draft_resource'], records=[record]) flash('Stembureau opgeslagen') return redirect(url_for('gemeente_stemlokalen_overzicht')) return render_template('gemeente-stemlokalen-edit.html', form=form, gemeente=gemeente, upload_deadline_passed=check_deadline_passed())
def gemeente_stemlokalen_dashboard(): # Select a gemeente if none is currently selected if not 'selected_gemeente_code' in session: return redirect(url_for('gemeente_selectie')) gemeente = Gemeente.query.filter_by( gemeente_code=session['selected_gemeente_code']).first() elections = gemeente.elections.all() # Pick the first election. In the case of multiple elections we only # retrieve the stembureaus of the first election as the records for # both elections are the same (at least the GR2018 + referendum # elections on March 21st 2018). verkiezing = elections[0].verkiezing all_publish_records = ckan.get_records( ckan.elections[verkiezing]['publish_resource']) all_draft_records = ckan.get_records( ckan.elections[verkiezing]['draft_resource']) gemeente_publish_records = [ record for record in all_publish_records['records'] if record['CBS gemeentecode'] == gemeente.gemeente_code ] gemeente_draft_records = [ record for record in all_draft_records['records'] if record['CBS gemeentecode'] == gemeente.gemeente_code ] _remove_id(gemeente_publish_records) _remove_id(gemeente_draft_records) toon_stembureaus_pagina = False if gemeente_publish_records: toon_stembureaus_pagina = True show_publish_note = False if gemeente_draft_records != gemeente_publish_records: show_publish_note = True vooringevuld = '' vooringevuld_fn = ( 'files/deels_vooringevuld/waarismijnstemlokaal.nl_invulformulier_%s_' 'deels_vooringevuld.xlsx' % (gemeente.gemeente_naam)) if os.path.exists(vooringevuld_fn): vooringevuld = vooringevuld_fn form = FileUploadForm() # Save, parse and validate an uploaded spreadsheet and save the # stembureaus if form.validate_on_submit(): f = form.data_file.data filename = secure_filename(f.filename) filename = '%s__%s' % (gemeente.gemeente_code, filename) file_path = os.path.join( os.path.abspath(os.path.join(app.instance_path, '../upload')), filename) f.save(file_path) parser = UploadFileParser() app.logger.info('Processing uploaded file for %s' % (gemeente.gemeente_naam)) try: records = parser.parse(file_path) except ValueError as e: app.logger.warning('Upload failed: %s' % e) flash( Markup( '<span class="text-red">Uploaden mislukt</span>. Het ' 'lijkt er op dat u geen gebruik maakt van (de meest ' 'recente versie van) de stembureau-spreadsheet. Download ' 'een <a href="/files/waarismijnstemlokaal.nl_' 'invulformulier.xlsx"><b>leeg</b></a> of <a href="%s"><b>' 'deels vooringevuld</b></a> stembureau-spreadsheet en vul ' 'de gegevens volgens de instructies in de spreadsheet in ' 'om deze vervolgens op deze pagina te ' 'uploaden.' % (vooringevuld))) return render_template( 'gemeente-stemlokalen-dashboard.html', verkiezing_string=_format_verkiezingen_string(elections), gemeente=gemeente, total_publish_records=len(gemeente_publish_records), total_draft_records=len(gemeente_draft_records), form=form, show_publish_note=show_publish_note, vooringevuld=vooringevuld, toon_stembureaus_pagina=toon_stembureaus_pagina, upload_deadline_passed=check_deadline_passed()) validator = Validator() results = validator.validate(records) # If the spreadsheet did not validate then return the errors as # flash messages if not results['no_errors']: flash( Markup( '<span class="text-red">Uploaden mislukt</span>. Los de ' 'hieronder getoonde foutmeldingen op en upload de ' 'spreadsheet opnieuw.' '<br><br>')) for column_number, col_result in sorted( results['results'].items()): if col_result['errors']: error_flash = ( '<b>Foutmelding(en) in <span class="text-red">' 'invulveld %s (oftewel kolom "%s")</span></b>:' % (column_number - 5, _colnum2string(column_number))) error_flash += '<ul>' for column_name, error in col_result['errors'].items(): error_flash += '<li>%s: %s</li>' % (column_name, error[0]) error_flash += '</ul><br>' flash(Markup(error_flash)) # If there not a single value in the results then state that we # could not find any stembureaus elif not results['found_any_record_with_values']: flash( Markup( '<span class="text-red">Uploaden mislukt</span>. Er zijn geen ' 'stembureaus gevonden in de spreadsheet.')) # If the spreadsheet did validate then first delete all current # stembureaus from the draft_resource and then save the newly # uploaded stembureaus to the draft_resources of each election # and finally redirect to the overzicht else: # Delete all stembureaus of current gemeente if gemeente_draft_records: for election in [x.verkiezing for x in elections]: ckan.delete_records( ckan.elections[election]['draft_resource'], {'CBS gemeentecode': gemeente.gemeente_code}) # Create and save records for election in [x.verkiezing for x in elections]: records = [] for _, result in results['results'].items(): if result['form']: records.append( _create_record(result['form'], result['uuid'], gemeente, election)) ckan.save_records(ckan.elections[election]['draft_resource'], records=records) flash( 'Het uploaden van stembureaus is gelukt! Controleer in het ' 'overzicht hieronder of alles klopt en voer eventuele ' 'wijzigingen door. Klik vervolgens op de "Publiceer"-knop als ' 'alles klopt.') return redirect(url_for('gemeente_stemlokalen_overzicht')) return render_template( 'gemeente-stemlokalen-dashboard.html', verkiezing_string=_format_verkiezingen_string(elections), gemeente=gemeente, total_publish_records=len(gemeente_publish_records), total_draft_records=len(gemeente_draft_records), form=form, show_publish_note=show_publish_note, vooringevuld=vooringevuld, toon_stembureaus_pagina=toon_stembureaus_pagina, upload_deadline_passed=check_deadline_passed())
def gemeente_stemlokalen_edit(stemlokaal_id=None): # Select a gemeente if none is currently selected if not 'selected_gemeente_code' in session: return redirect(url_for('gemeente_selectie')) gemeente = Gemeente.query.filter_by( gemeente_code=session['selected_gemeente_code']).first() elections = gemeente.elections.all() # Need this to get a starting point for the clickmap; # Uses re.sub to remove provinces from some gemeenten which is how we write # gemeenten in WIMS, but which are not used in the BAG, e.g. 'Beek (L.)', # but keep 'Bergen (NH.)' and 'Bergen (L.)' as the BAG also uses that # spelling. # TODO this won't work for BES-eilanden as they don't exist in the BAG, so # exclude them from the filter below and initialize a custom bag_record # with coordinates for the BES-eilanden. bag_record = BAG.query.filter_by( gemeente=gemeente.gemeente_naam if 'Bergen (' in gemeente.gemeente_naam else re.sub(' \(.*\)$', '', gemeente.gemeente_naam)).order_by( 'openbareruimte').first() # Pick the first election. In the case of multiple elections we only # retrieve the stembureaus of the first election as the records for # both elections are the same (at least for the GR2018 + referendum # elections on March 21st 2018). verkiezing = elections[0].verkiezing all_draft_records = ckan.get_records( ckan.elections[verkiezing]['draft_resource']) gemeente_draft_records = [ record for record in all_draft_records['records'] if record['CBS gemeentecode'] == gemeente.gemeente_code ] # Initialize the form with the data already available in the draft init_record = {} if stemlokaal_id: for record in gemeente_draft_records: if record['UUID'] == stemlokaal_id: # Split the Verkiezingen attribute into a list if record['Verkiezingen']: record['Verkiezingen'] = [ x.strip() for x in record['Verkiezingen'].split(';') ] init_record = Record( **{k.lower(): v for k, v in record.items()}).record form = EditForm(**init_record) # When the user clicked the 'Annuleren' button go back to the # overzicht page without doing anything if form.submit_annuleren.data: flash('Bewerking geannuleerd') return redirect(url_for('gemeente_stemlokalen_overzicht')) # When the user clicked the 'Verwijderen' button delete the # stembureau from the draft_resources of each election if form.submit_verwijderen.data: if stemlokaal_id: for election in [x.verkiezing for x in elections]: ckan.delete_records(ckan.elections[election]['draft_resource'], {'UUID': stemlokaal_id}) flash('Stembureau verwijderd') return redirect(url_for('gemeente_stemlokalen_overzicht')) # When the user clicked the 'Opslaan' button save the stembureau # to the draft_resources of each election if form.validate_on_submit(): if not stemlokaal_id: stemlokaal_id = uuid.uuid4().hex for election in [x.verkiezing for x in elections]: record = _create_record(form, stemlokaal_id, gemeente, election) ckan.save_records(ckan.elections[election]['draft_resource'], records=[record]) flash('Stembureau opgeslagen') return redirect(url_for('gemeente_stemlokalen_overzicht')) return render_template('gemeente-stemlokalen-edit.html', form=form, gemeente=gemeente, bag_record=bag_record, upload_deadline_passed=check_deadline_passed())
def test_datastore_upsert(resource_id): """ Insert or update data in the datastore table in a resource with an example record; used for testing """ record = { "Gemeente": "'s-Gravenhage", "CBS gemeentecode": "GM0518", "Nummer stembureau": "517", "Naam stembureau": "Stadhuis", "Gebruiksdoel van het gebouw": "kantoor", "Website locatie": ("https://www.denhaag.nl/nl/bestuur-en-organisatie/contact-met-" "de-gemeente/stadhuis-den-haag.htm"), "Wijknaam": "Centrum", "CBS wijknummer": "WK051828", "Buurtnaam": "Kortenbos", "CBS buurtnummer": "BU05182811", "BAG Nummeraanduiding ID": "0518100000275247", "Straatnaam": "Spui", "Huisnummer": 70, "Huisletter": "", "Huisnummertoevoeging": "", "Postcode": "2511 BT", "Plaats": "Den Haag", "Extra adresaanduiding": "Ingang aan achterkant gebouw", "X": 81611, "Y": 454909, "Latitude": 52.0775912, "Longitude": 4.3166395, "Openingstijden 14-03-2022": "2022-03-14T07:30:00 tot 2022-03-14T21:00:00", "Openingstijden 15-03-2022": "2022-03-15T07:30:00 tot 2022-03-15T21:00:00", "Openingstijden 16-03-2022": "2022-03-16T07:30:00 tot 2022-03-16T21:00:00", "Toegankelijk voor mensen met een lichamelijke beperking": "ja", "Akoestiek": "ja", "Auditieve hulpmiddelen": "gebarentolk", "Visuele hulpmiddelen": "leesloep, stemmal, vrijwilliger/host aanwezig", "Gehandicaptentoilet": "nee", "Kieskring ID": "'s-Gravenhage", "Hoofdstembureau": "Nederland", "Tellocatie": "ja", "Contactgegevens gemeente": "Unit Verkiezingen, [email protected] 070-3534488 Gemeente Den Haag Publiekszaken/Unit Verkiezingen Postbus 84008 2508 AA Den Haag", "Verkiezingswebsite gemeente": "https://www.stembureausindenhaag.nl/", #"Verkiezingen": "", "ID": "NLODSGM0518stembureaus20220316009", "UUID": uuid.uuid4().hex } ckan.save_records(resource_id=resource_id, records=[record])