def __init__(self, conres, ruling_parties=RULING_PARTIES): self.conres = conres self.con = conres.constituency self.winner_slug = slugify(conres.winning_party) self.runner_up_slug = slugify(conres.results[1].party) self.full_region = self.con.country_and_region self.region_slug = slugify(self.full_region) # Ensures this is populated for W, S, NI self.region = self.con.region or self.con.country self.winning_margin = conres.margin_pc if not ABSOLUTE_MARGIN_PC and conres.winning_party not in ruling_parties: self.winning_margin = -conres.margin_pc self.winner_votes = conres.results[0].valid_votes self.runner_up_votes = conres.results[1].valid_votes # Do all the percentage calculations in Python using Decimal to avoid # horrible rounding/floating-point issues in JS self.turnout = '%.1f' % (conres.turnout_pc) self.winner_pc = '%.1f' % (100 * self.winner_votes / self.con.valid_votes) self.runner_up_pc = '%.1f' % (100 * self.runner_up_votes / self.con.valid_votes) self.won_by_pc = '%.1f' % (100 * (self.winner_votes - self.runner_up_votes) / self.con.valid_votes)
def constituency_name_to_region(region_data, slugify_constituency_name=True, key_mappings=None): """ Reverse mapping of the output from load_region_data() i.e. constituency name->region By default the constituency name key is slugified. Alternatively, if key_mappings is a dict of str->list, the returned dict will have all constituency variant IDs mapped to the region e.g. * Constituency name * Slugified constituency name * ONS Code * PA number But all this is entirely dependent on what is in key_mappings - which can be set up with constituency_id_mappings """ con_to_region = {} # reverse map constituency name for reg, con_list in region_data.items(): for con in con_list: if key_mappings: try: constituency_ids = key_mappings[con] except KeyError: constituency_ids = key_mappings[slugify(con)] for k in constituency_ids: con_to_region[k] = reg else: k = (slugify(con) if slugify_constituency_name else con) con_to_region[k] = reg return con_to_region
def load_constituencies_from_admin_csv(admin_csv, con_to_region_map, quiet=False): """ Return a list of Constituency objects """ # ons_to_con_map = {} regions = set(con_to_region_map.values()) slug_to_canonical_region_map = dict((slugify(r), r) for r in regions) ret = [] skippable_lines = sniff_admin_csv_for_ignorable_lines(admin_csv) with open(admin_csv, 'r', **csv_reader_kwargs) as inputstream: # Ignore the first two rows, the useful headings are on the third row for _ in range(skippable_lines): xxx = inputstream.readline() reader = csv.DictReader(inputstream) for i, row in enumerate(reader): # pdb.set_trace() # Note extraneous trailing space on 'Electorate ' :-( # print("%d %s %s" % (i, row['Constituency'], row['Electorate '])) if not is_blank_row(row): con = Constituency(row) if not con.region: con.region = get_region_for_constituency(con, con_to_region_map, quiet) ORIG = """ if con.country == 'England': slug_con = slugify(con.name) try: con.region = con_to_region_map[slug_con] except KeyError as err: logging.error('No region found for %s/%s' % (con.name, slug_con)) """ else: con.region = slug_to_canonical_region_map[slugify(con.region)] ret.append(con) # return ons_to_con_map return ret
def get_region_for_constituency(con, con_to_region_map, quiet=False): """ Given a Constituency object, use any of the various unique IDs to get a region from the provided dictionary. """ if con.country and con.country != 'England': return con.country poss_keys = [str(getattr(con, prop)) for prop in ('name', 'ons_code', 'pa_number')] for val in poss_keys: try: return con_to_region_map[val] except KeyError: pass if not quiet: logging.warning('No region found for %s, trying slugified name...' % (poss_keys)) slug = slugify(con.name) return con_to_region_map[slug]
def constituency_id_mappings(constituency_csv=DEFAULT_CONSTITUENCY_CSV): """ Return a dict mapping various unique identifiers for a constituency to the set (well a list) of all those Ids) """ # basic_regions = load_region_data(add_on_countries=True) basic_regions = load_region_data() con_to_region = constituency_name_to_region(basic_regions) ge2017_data = load_constituencies_from_admin_csv(constituency_csv, con_to_region, quiet=True) c2ids_map = {} for con in ge2017_data: ids = [con.name, slugify(con.name), con.ons_code] if con.pa_number: # Keep consistent with other values' type to allow for sorting ids.append(str(con.pa_number)) # c2ids_map[con.name] = ids # Blows up on Weston-[Ss]uper-Mare, so use all IDs for id in ids: c2ids_map[id] = ids return c2ids_map
def process(petition_file=None, html_output=True, include_all=True, output_function=py2print, embed=True, petition_data=None): with open(os.path.join('intermediate_data', 'regions.json')) as regionstream: regions = json.load(regionstream) euref_data = load_and_process_euref_data() election_data = load_and_process_data(ADMIN_CSV, RESULTS_CSV, regions, euref_data) if petition_file: petition_data = load_petition_data(petition_file) signature_count, petition_timestamp, constituency_data = process_petition_data(petition_data) constituency_total = sum([z for z in constituency_data.values()]) counter = 0 include_all = True sig_above_margin = 0 pro_leave_sig_above_margin = 0 for conres in election_data: ons_code = conres.constituency.ons_code if constituency_data[ons_code] > conres.winning_margin: sig_above_margin += 1 if euref_data[ons_code].leave_pc > 50.0: pro_leave_sig_above_margin += 1 if html_output: html_header(output_function, embed) sub_header(output_function, petition_timestamp, signature_count, constituency_total, sig_above_margin, pro_leave_sig_above_margin) for i, conres in enumerate(election_data, 1): margin = conres.winning_margin ons_code = conres.constituency.ons_code sigs = constituency_data[ons_code] leave_pc = '%2d%%%s' % ( euref_data[ons_code].leave_pc, ' ' if euref_data[ons_code].known_result else '*') slug_party = slugify(conres.winning_party) # output_function(slug_party) #margin_text = '%sGE2017 Winning margin: %5d votes%s ' % \ # (PARTY_COLOURS[slug_party], margin, COLORAMA_RESET) if sigs > margin or include_all: cells = [] counter += 1 # cells.append(('numeric', '%s' % (counter))) Now done with CSS magic classes = ['small centred'] if conres.constituency.country: classes.append('country-%s' % slugify(conres.constituency.country)) if conres.constituency.region: classes.append('region-%s' % slugify(conres.constituency.region)) cells.append((' '.join(classes), conres.constituency.country_and_region)) con_name = conres.constituency.name cells.append(('', '<a class="plain" href="#%s">%s</a>' % (slugify(con_name), con_name))) if euref_data[ons_code].leave_pc >= 55.0: kls = 'voted-leave-55' elif euref_data[ons_code].leave_pc >= 50.0: kls = 'voted-leave-50' elif euref_data[ons_code].leave_pc >= 45.0: kls = 'voted-leave-45' else: kls = '' cells.append(('numeric ' + kls, leave_pc)) cells.append(('party-%s numeric' % slug_party, '%d' % (conres.winning_result.valid_votes))) cells.append(('party-%s numeric' % slug_party, '%d' % margin)) cells.append(('numeric', '%s' % sigs)) sig_pc = 100 * sigs / conres.constituency.electorate sig_cls = 'signed-%d' % (min(50, int(sig_pc / 5) * 5)) cells.append(('numeric %s' % (sig_cls), '%.1f%%' % sig_pc)) # NB: valid_votes is probably slightly less than turnout sig_vs_turnout_pc = 100 * sigs / conres.constituency.valid_votes ratio_class = 'signed-%d' % (min(50, int(sig_vs_turnout_pc / 5) * 5)) cells.append(('numeric %s' % (ratio_class), '%d%%' % (sig_vs_turnout_pc))) sig_vs_winner_pc = 100 * sigs /conres.winning_result.valid_votes ratio_range = int(sig_vs_winner_pc / 10) * 10 ratio_class = 'threshold-%d' % (min(100, ratio_range)) cells.append(('numeric %s' % (ratio_class), '%d%%' % (sig_vs_winner_pc))) ratio = (100 * sigs / margin) ratio_range = int(ratio / 10) * 10 ratio_class = 'threshold-%d' % (min(100, ratio_range)) cells.append(('numeric %s' % (ratio_class), '%d%%' % ratio)) cell_str =''.join(['<td class="%s">%s</td>' % z for z in cells]) output_function('<tr id="%s">%s</tr>' % (slugify(con_name), cell_str)) output_function('''</table></body>\n</html>\n''') else: output_function('Based on petition data at %s (%d signatures)' % (petition_timestamp, signature_count)) output_function('Asterisked vote leave percentages are estimates - see %s' % EUREF_VOTES_BY_CONSTITUENCY_SHORT_URL) for i, conres in enumerate(election_data, 1): margin = conres.winning_margin ons_code = conres.constituency.ons_code sigs = constituency_data[ons_code] leave_pc = '%s%2d%%%s%s' % ( Back.MAGENTA if euref_data[ons_code].leave_pc >= 50.0 else '', euref_data[ons_code].leave_pc, ' ' if euref_data[ons_code].known_result else '*', COLORAMA_RESET) slug_party = slugify(conres.winning_party) # output_function(slug_party) margin_text = '%sGE2017 Winning margin: %5d votes%s ' % \ (PARTY_COLOURS[slug_party], margin, COLORAMA_RESET) if sigs > margin or include_all: counter += 1 output_function('%3d. %-45s : Voted leave: %s %s Current signatures: %5d' % (counter, conres.constituency.name, leave_pc, margin_text, sigs))
def output_svg(out, data, sort_method='by_region'): out.write(f'''<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="{OVERALL_WIDTH}" height="{OVERALL_HEIGHT}" id="election-bars" class="js-disabled"> ''') out.write('''<style type="text/css">\n<![CDATA[\n''') output_file(out, os.path.join(STATIC_DIR, 'party_and_region_colours.css')) output_file(out, os.path.join(STATIC_DIR, 'misc.css')) output_file(out, os.path.join(STATIC_DIR, 'euref_ge_comparison.css')) out.write(']]>\n </style>\n') if DEBUG_MODE: out.write(f'''<rect x="3" y="3" width="{OVERALL_WIDTH-10}" height="{OVERALL_HEIGHT-10}" class="debug2" />''') output_file(out, os.path.join(INCLUDES_DIR, 'ge_constituency_bar_chart_static.svg')) out.write(f'<text x="50" y="200">{sort_method}</text>\n') MARGIN = 10 COLUMN_WIDTH=5 Y_FACTOR = 100 TOTAL_HEIGHT = 1600 COUNTRY_KEY_HEIGHT = 50 REGION_KEY_HEIGHT = 50 CONSTITUENCY_Y_OFFSET = TOTAL_HEIGHT - MARGIN - COUNTRY_KEY_HEIGHT - \ REGION_KEY_HEIGHT - MARGIN out.write('<g id="datapoints">\n') for i, conres in enumerate(sorted(election_data, key=SORT_OPTIONS[sort_method])): ecr = EnhancedConstituencyResult(conres) con = conres.constituency slugified_country = slugify(con.country) x_offset = MARGIN + (i * COLUMN_WIDTH) height = int(con.electorate / Y_FACTOR) y_offset = CONSTITUENCY_Y_OFFSET - height out.write(f'<g class="constituency" %s>\n' % (ecr.data_attributes_string())) out.write(f'<rect x="{x_offset}" y="{y_offset}" ' f'width="{COLUMN_WIDTH}" height="{height}" ' f'class="constituency total-electorate" ' f'title="{con.name}" />\n') country_y_pos = CONSTITUENCY_Y_OFFSET + MARGIN out.write(f'<rect x="{x_offset}" y="{country_y_pos}" ' f'width="{COLUMN_WIDTH}" height="{COUNTRY_KEY_HEIGHT}" ' f'class="country-{slugified_country}" ' f'title="{con.name} - {con.country}" />\n') region_y_pos = country_y_pos + COUNTRY_KEY_HEIGHT if con.region: region_class = 'region-' + slugify(con.region) region_text = con.region else: region_class = 'country-' + slugified_country region_text = con.country out.write(f'<rect x="{x_offset}" y="{region_y_pos}" ' f'width="{COLUMN_WIDTH}" height="{REGION_KEY_HEIGHT}" ' f'class="{region_class}" ' f'title="{con.name} - {region_text}" />\n') party_y_offset = CONSTITUENCY_Y_OFFSET for res_num, res in enumerate(conres.results): slugified_party = slugify(res.party) if res_num > 1: slugified_party = 'loser' party_height = int(res.valid_votes / Y_FACTOR) party_y_offset -= party_height out.write(f' <rect x="{x_offset}" y="{party_y_offset}" ' f'width="{COLUMN_WIDTH}" height="{party_height}" ' f'class="result constituency party-{slugified_party}" />\n') out.write(f'</g> <!-- end of g.constituency -->\n') out.write(f'</g> <!-- end of #datapoints -->\n') out.write('<script type="text/ecmascript">\n<![CDATA[\n') output_file(out, os.path.join(STATIC_DIR, 'constituency_details.js')) # TODO: get the brexit_regions functionality working for these bar charts # output_file(out, os.path.join(STATIC_DIR, 'brexit_regions.js')) out.write('document.querySelector("svg").classList.remove("js-disabled");\n') out.write('setupConstituencyDetails("hover-details", "#datapoints g.constituency");\n'); out.write('\n]]>\n</script>') out.write('</svg>\n')
def output_svg(out, data, year, ruling_parties=RULING_PARTIES, value_map=None): out.write(f'''<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="{OVERALL_WIDTH}" height="{OVERALL_HEIGHT}" id="election" class="js-disabled"> ''') out.write( f'<title>Constituency results in {year} General Election and 2016 EU Referendum</title>\n' ) out.write( '<description>Scatter plot of constituency results ' f'in {year} General Election and 2016 EU Referendum</description>\n') out.write('''<style type="text/css">\n<![CDATA[\n''') output_file(out, os.path.join(STATIC_DIR, 'party_and_region_colours.css')) output_file(out, os.path.join(STATIC_DIR, 'misc.css')) output_file(out, os.path.join(STATIC_DIR, 'euref_ge_comparison.css')) out.write(']]>\n </style>\n') if DEBUG_MODE: out.write( f'''<rect x="3" y="3" width="{OVERALL_WIDTH-10}" height="{OVERALL_HEIGHT-10}" class="debug2" />''') if not value_map: value_map = {} output_file(out, os.path.join(INCLUDES_DIR, 'brexit_fptp_static.svg'), value_map=value_map) # So leave covered a range of (80-20)*10 = 600 pixels (vertical) # and GE margin covered a range of 110*10 = 1100 pixels (vertical) # so increase from 10 to 15 now that we are using Full HD(ish) resolution # (15 is just a bit too big, as is 14 on some browsers e.g. FF # Update: 14 is too much for X now that I calculate GE margin properly LEAVE_PC_SCALE = 13 MARGIN_PC_SCALE = 9 LEAVE_PC_RANGE = (20, 80) # inclusive range # NB: in GE2017, there are 3 Labour constituencies with a margin between # 50 and 55%, so watch out in future in case this goes higher. (No Tory # constituency goes above 40%) # UPDATE: due to using electorate rather than valid vote, the range is more # like 80, not 55 if ABSOLUTE_MARGIN_PC: MARGIN_PC_RANGE = (0, 80) # inclusive range else: # inclusive range, we only need to ~50 for Tories, but prefer to keep # symmetrical MARGIN_PC_RANGE = (-80, 80) def calculate_fptp_margin_offset(val): return CENTRE_X + (val * MARGIN_PC_SCALE) def calculate_leave_pc_offset(val): return CENTRE_Y - ((val - 50) * LEAVE_PC_SCALE) ### Horizontal grid lines and Y axis x_start = calculate_fptp_margin_offset(MARGIN_PC_RANGE[0]) x_end = calculate_fptp_margin_offset(MARGIN_PC_RANGE[1]) for val in range(LEAVE_PC_RANGE[0], LEAVE_PC_RANGE[1] + 1): y_offset = calculate_leave_pc_offset(val) if val % 5 != 0: css_class = 'grid-line-light' else: if val == 50: css_class = 'grid-line-strong' else: css_class = 'grid-line' out.write( f'<text x="{x_start-2}" y="{y_offset}" class="y-axis-label">' f'{val}%</text>') out.write(f'<line x1="{x_start}" y1="{y_offset}" ' f'x2="{x_end}" y2="{y_offset}" class="{css_class}" />\n') out.write(f'<text x="{x_start-30}" y="{y_offset}" class="y-axis-label">' f'Voted leave</text>') ### Vertical grid lines and X axis labels y_start = calculate_leave_pc_offset(LEAVE_PC_RANGE[0]) y_end = calculate_leave_pc_offset(LEAVE_PC_RANGE[1]) for val in range(MARGIN_PC_RANGE[0], MARGIN_PC_RANGE[1] + 1): x_offset = calculate_fptp_margin_offset(val) if val % 5 != 0: css_class = 'grid-line-light' else: out.write( f'<text x="{x_offset}" y="{y_start+10}" class="x-axis-label">' f'{val}%</text>') if not ABSOLUTE_MARGIN_PC and val == 0: css_class = 'grid-line-strong' else: css_class = 'grid-line' out.write(f'<line x1="{x_offset}" y1="{y_start}" ' f'x2="{x_offset}" y2="{y_end}" class="{css_class}" />\n') out.write( f'<text x="{CENTRE_X}" y="{y_start+20}" class="y-axis-label centre-aligned">' f'Winning margin in General Election</text>') relevant_parties = set() regions = set() out.write('<g id="datapoints">\n') prev_region = None for i, conres in enumerate( sorted(election_data, key=lambda z: z.constituency.country_and_region)): ecr = EnhancedConstituencyResult(conres, ruling_parties=ruling_parties) con = conres.constituency winner = slugify(conres.winning_party) runner_up = slugify(conres.results[1].party) relevant_parties.update( [conres.winning_party, conres.results[1].party]) full_region = con.country_and_region regions.add(full_region) slugified_region = slugify(full_region) region = con.region or con.country # Ensure this is populated for W, S, NI if prev_region is None or slugified_region != prev_region: if prev_region is not None: out.write(f'</g> <!-- end of {prev_region} -->\n') out.write(f'''<g class="js-level level-{slugified_region}" id="region-{slugified_region}" data-country="{con.country}" data-region="{region}">\n''') prev_region = slugified_region y_offset = calculate_leave_pc_offset(con.euref.leave_pc) winning_margin = conres.margin_pc if not ABSOLUTE_MARGIN_PC and conres.winning_party not in ruling_parties: winning_margin = -conres.margin_pc x_offset = Decimal('%.1f' % (calculate_fptp_margin_offset(winning_margin))) if con.euref.known_result: x_offset -= DOT_RADIUS y_offset -= DOT_RADIUS out.write(f'<rect x="{x_offset}" y="{y_offset}" ' f'width="{DOT_DIAMETER}" height="{DOT_DIAMETER}" ') else: out.write(f'<circle cx="{x_offset}" cy="{y_offset}" ' f'r="{DOT_RADIUS}" ') out.write( f'class="constituency party-{winner} second-place-{runner_up}"\n') out.write(ecr.data_attributes_string() + '/>\n') out.write(f'</g> <!-- end of {prev_region} -->\n') out.write(f'</g> <!-- end of #datapoints -->\n') ### Legend x_pos = 1575 y_pos = 125 line_spacing = 14 DEFAULT_BUTTON_WIDTH = 190 out.write('<g class="legend">\n') out.write( f'<text x="{x_pos}" y="{y_pos}" class="heading">Legend and filters</text>\n' ) for txt in [ 'Fill colour indicates winner. Border/', 'outline colour indicates runner-up.', '', 'Square dots indicate definite known', 'EU Referendum constituency results', 'circles indicate estimated results', 'as published on Parliament.uk.', '', 'Click on the items below to filter', 'on that particular party, winner,', 'party, winner, runner-up or region.', ]: y_pos += line_spacing out.write(f'<text x="{x_pos}" y="{y_pos}">{txt}</text>\n') def output_button_with_arbitrary_content(out, x_offset, y_offset, line_spacing, svg_content, element_id=None, classes=None, data_attributes=None, width=DEFAULT_BUTTON_WIDTH, is_selected=False): if not classes: classes = [] if not data_attributes: data_attributes = {} if is_selected: classes.append(' selected') classes.append('js-button') if not element_id: id_bit = '' else: id_bit = f'id="{element_id}"' class_string = ' '.join(classes) def data_prefixed(attribute_name): if not attribute_name.startswith('data-'): return 'data-%s' % (attribute_name) else: return attribute_name attribute_bits = [ '%s="%s"' % (data_prefixed(k), v) for k, v in data_attributes.items() ] attribute_string = ' '.join(attribute_bits) out.write(f'''<g class="{class_string}" {id_bit} {attribute_string}>\n''') out.write( f''' <rect x="{x_offset}" y="{y_offset}" rx="{line_spacing * 0.6}" width="{width}" height="{line_spacing * 1.25}" />\n''') out.write(svg_content) out.write('</g>\n') def output_text_button(out, x_offset, y_offset, line_spacing, label, element_id=None, classes=None, data_attributes=None, width=DEFAULT_BUTTON_WIDTH, is_selected=False): # -2 is a hack to allow for descenders svg_bits = f' <text x="{x_offset+10}" y="{y_offset+line_spacing-2}">{label}</text>\n' return output_button_with_arbitrary_content(out, x_offset, y_offset, line_spacing, svg_bits, element_id, classes, data_attributes, width, is_selected) # y_pos += line_spacing for p in sorted(relevant_parties): p_slug = slugify(p) y_pos += line_spacing * 1.5 circle_svg = f'''<circle cx="{x_pos+15}" cy="{y_pos+9}" r="4" class="constituency winner party-{p_slug}" />\n''' output_button_with_arbitrary_content( out, x_pos, y_pos, line_spacing, circle_svg, # 'js-party-filter-%s' % (p_slug), classes=['selectable-party-position'], data_attributes={'party': p_slug}, width=30) circle_svg = f'''<circle cx="{x_pos+50}" cy="{y_pos+9}" r="4" class="constituency second-place second-place-{p_slug}" />\n''' output_button_with_arbitrary_content( out, x_pos + 35, y_pos, line_spacing, circle_svg, # 'js-party-filter-%s' % (p_slug), classes=['selectable-party-position'], data_attributes={'party': p_slug}, width=30) output_text_button( out, x_pos + 70, y_pos, line_spacing, p, # 'js-party-filter-%s' % (p_slug), classes=['selectable-party'], data_attributes={'party': p_slug}, width=120) y_pos += 30 # 40 is OK for 2017, but other GEs have more parties out.write('<g class="hide-if-no-js">\n') output_text_button(out, x_pos, y_pos, line_spacing, 'All regions', element_id='js-level-all', classes=['js-level-button'], data_attributes={'level': 'all'}, is_selected=True) for region in sorted(regions): y_pos += line_spacing * 1.5 slugified_region = slugify(region) output_text_button(out, x_pos, y_pos, line_spacing, short_region(region), element_id='js-level-%s' % slugified_region, classes=['js-level-button'], data_attributes={'level': slugified_region}) out.write('</g> <!-- end of hide-if-no-js -->\n') y_pos = 900 output_text_button(out, x_pos, y_pos, line_spacing, 'Toggle dark mode', element_id='js-dark-mode-toggle') out.write('</g> <!-- end of legend -->\n') out.write('<script type="text/ecmascript">\n<![CDATA[\n') output_file(out, os.path.join(STATIC_DIR, 'constituency_details.js')) output_file(out, os.path.join(STATIC_DIR, 'brexit_regions.js')) out.write( 'document.querySelector("svg").classList.remove("js-disabled");\n') out.write( 'setupConstituencyDetails("hover-details", "#datapoints .constituency");\n' ) out.write('\n]]>\n</script>') out.write('</svg>\n')