def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: # each election has a statewide and a county pre-processed file. statewide_filename = self._generate_statewide_filename(election) meta.append({ "generated_filename": statewide_filename, "raw_url": election['direct_links'][0], "pre_processed_url": build_raw_github_url(self.state, str(year), statewide_filename), "ocd_id": 'ocd-division/country:us/state:tx', "name": 'Texas', "election": election['slug'] }) county_filename = self._generate_county_filename(election) meta.append({ "generated_filename": county_filename, "raw_url": election['direct_links'][0], "pre_processed_url": build_raw_github_url(self.state, str(year), county_filename), "ocd_id": 'ocd-division/country:us/state:tx', "name": 'Texas', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: # parish-level file jurisdiction = 'ocd-division/country:us/state:la' generated_filename = self._generate_parish_filename(election) meta.append({ "generated_filename": generated_filename, "raw_url": election['portal_link'], "pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename), "ocd_id": jurisdiction, "name": 'Louisiana', "election": election['slug'] }) # precinct-level file jurisdiction = 'ocd-division/country:us/state:la' generated_filename = self._generate_precinct_filename(election) meta.append({ "generated_filename": generated_filename, "raw_url": election['portal_link'], "pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename), "ocd_id": jurisdiction, "name": 'Louisiana', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if 'special' in election['slug']: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True ] for result in results: generated_filename = result['path'] if result['county']: ocd_id = 'ocd-division/country:us/state:mo/county:' + result[ 'county'].replace(' ', '_').lower() else: ocd_id = 'ocd-division/country:us/state:mo' meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, str(year), result['path']), "ocd_id": ocd_id, "name": 'Missouri', "election": election['slug'] }) else: # primary, general and runoff statewide elections have 1 or 2 files per county # some general runoffs will have smaller numbers of files results = [ x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False ] for result in results: ocd_id = 'ocd-division/country:us/state:mo' generated_filename = result['path'] meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, str(year), result['path']), "ocd_id": ocd_id, "name": 'Missouri', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if 'special' in election['slug']: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True] for result in results: generated_filename = result['path'] jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0] ocd_id = jurisdiction['ocd_id'] name = jurisdiction['jurisdiction'] meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, str(year), result['path']), "ocd_id": ocd_id, "name": name, "election": election['slug'] }) else: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] == ''] for result in results: if result['jurisdiction'] == '': jurisdiction = 'ocd-division/country:us/state:nv' generated_filename = self._generate_statewide_filename(result) pre_processed_url = None else: jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]['ocd_id'] generated_filename = result['path'] pre_processed_url = build_raw_github_url(self.state, str(year), result['path']) meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": pre_processed_url, "ocd_id": jurisdiction, "name": result['jurisdiction'], "election": election['slug'] }) results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] != ''] for result in results: jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]['ocd_id'] generated_filename = result['path'] pre_processed_url = build_raw_github_url(self.state, str(year), result['path']) meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": pre_processed_url, "ocd_id": jurisdiction, "name": result['jurisdiction'], "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if election['special']: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True] else: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False] for result in results: if result['url']: raw_url = result['url'] else: raw_url = None if result['county'] == '': generated_filename = self._generate_filename(election['start_date'], election['race_type'], result) ocd_id = 'ocd-division/country:us/state:or' name = "Oregon" else: generated_filename = self._generate_county_filename(election['start_date'], result) ocd_id = 'ocd-division/country:us/state:or/county:%s' % result['county'].lower().replace(" ", "_") name = result['county'] meta.append({ "generated_filename": generated_filename, "raw_url": raw_url, "pre_processed_url": build_raw_github_url(self.state, election['start_date'][0:4], generated_filename), "ocd_id": ocd_id, "name": name, "election": election['slug'] }) # generate precinct files return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if 'special' in election['slug']: generated_filename = self._generate_filename(election, 'csv') meta.append({ "generated_filename": generated_filename, "raw_url": election['direct_links'][0], "pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename), "ocd_id": 'ocd-division/country:us/state:ri', "name": 'Rhode Island', "election": election['slug'] }) else: result = [x for x in self._url_paths() if x['date'] == election['start_date']][0] if result['raw_extracted_filename']: format = result['raw_extracted_filename'].split('.')[1] else: format = 'csv' meta.append({ "generated_filename": self._generate_filename(election, format), "raw_url": election['direct_links'][0], "raw_extracted_filename": result['raw_extracted_filename'], "pre_processed_url": None, "ocd_id": 'ocd-division/country:us/state:ri', "name": 'Rhode Island', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) if year < 2008: for election in elections: results = [x for x in self._url_paths() if x['date'] == election['start_date']] for result in results: generated_filename = self._generate_office_filename(election['direct_links'][0], election['start_date'], election['race_type'], result) meta.append({ "generated_filename": generated_filename, "raw_url": self._build_raw_url(year, result['path']), "pre_processed_url": build_github_url(self.state, generated_filename), "ocd_id": 'ocd-division/country:us/state:wv', "name": 'West Virginia', "election": election['slug'] }) elif year < 2012: for election in elections: meta.append({ "generated_filename": self._generate_statewide_filename(election), "raw_url": election['direct_links'][0], "pre_processed_url": None, "ocd_id": 'ocd-division/country:us/state:wv', "name": 'West Virginia', "election": election['slug'] }) else: for election in elections: csv_links = self._find_csv_links(election['direct_links'][0]) counties = self._jurisdictions() meta.append({ "generated_filename": self._generate_statewide_filename(election), "pre_processed_url": None, "raw_url": election['direct_links'][1], "ocd_id": 'ocd-division/country:us/state:wv', "name": 'West Virginia', "election": election['slug'] }) results = list(zip(counties, csv_links[1:])) for result in results: if election['start_date'] == "2016-11-08" and any(county == result[0]['county'] for county in ['Kanawha', 'Marshall', 'Nicholas', 'Cabell']): meta.append({ "generated_filename": self._generate_county_filename(result[0]['county'], election), "pre_processed_url": build_raw_github_url(self.state, '2016', self._generate_county_filename(result[0]['county'], election)), "raw_url": result[1], "ocd_id": result[0]['ocd_id'], "name": result[0]['county'], "election": election['slug'] }) else: meta.append({ "generated_filename": self._generate_county_filename(result[0]['county'], election), "pre_processed_url": None, "raw_url": result[1], "ocd_id": result[0]['ocd_id'], "name": result[0]['county'], "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if election['special']: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True ] else: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False ] for result in results: if result['url']: raw_url = result['url'] else: raw_url = None if result['county'] == '': generated_filename = self._generate_filename( election['start_date'], election['race_type'], result) ocd_id = 'ocd-division/country:us/state:or' name = "Oregon" else: generated_filename = self._generate_county_filename( election['start_date'], result) ocd_id = 'ocd-division/country:us/state:or/county:%s' % result[ 'county'].lower().replace(" ", "_") name = result['county'] meta.append({ "generated_filename": generated_filename, "raw_url": raw_url, "pre_processed_url": build_raw_github_url(self.state, election['start_date'][0:4], generated_filename), "ocd_id": ocd_id, "name": name, "election": election['slug'] }) # generate precinct files return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if 'special' in election['slug']: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True] for result in results: jurisdiction = 'ocd-division/country:us/state:nv' generated_filename = self._generate_statewide_precinct_filename(result) pre_processed_url = None meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": None, "ocd_id": jurisdiction, "name": 'Nevada', "election": election['slug'] }) else: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] == ''] for result in results: if result['jurisdiction'] == '': jurisdiction = 'ocd-division/country:us/state:nv' generated_filename = self._generate_statewide_filename(result) pre_processed_url = None else: jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]['ocd_id'] generated_filename = result['path'] pre_processed_url = build_raw_github_url(self.state, str(year), result['path']) meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": pre_processed_url, "ocd_id": jurisdiction, "name": result['jurisdiction'], "election": election['slug'] }) results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] != ''] for result in results: jurisdiction = 'ocd-division/country:us/state:nv' generated_filename = self._generate_statewide_precinct_filename(result) pre_processed_url = None meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": pre_processed_url, "ocd_id": jurisdiction, "name": 'Nevada', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if 'special' in election['slug']: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True] for result in results: generated_filename = result['path'] if result['county']: ocd_id = 'ocd-division/country:us/state:mo/county:' + result['county'].replace(' ','_').lower() else: ocd_id = 'ocd-division/country:us/state:mo' meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, str(year), result['path']), "ocd_id": ocd_id, "name": 'Missouri', "election": election['slug'] }) else: # primary, general and runoff statewide elections have 1 or 2 files per county # some general runoffs will have smaller numbers of files results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False] for result in results: ocd_id = 'ocd-division/country:us/state:mo' generated_filename = result['path'] meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, str(year), result['path']), "ocd_id": ocd_id, "name": 'Missouri', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] for election in elections: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and not election['special']] for result in results: meta.append({ "generated_filename": self._standardized_filename(election), "raw_url": result['url'], "pre_processed_url": build_raw_github_url( self.state, year, result['path']), "ocd_id": 'ocd-division/country:us/state:in', "name": 'Indiana', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: try: raw_url = election['direct_links'][0] # In reality, the election may have multiple source files, but we shouldn't be using the raw_url for anything except IndexError: raw_url = election['direct_link'] generated_filename = self._generate_filename(election) ocd_id = 'ocd-division/country:us/state:wi' name = "Wisconsin" meta.append({ "generated_filename": generated_filename, "raw_url": raw_url, "pre_processed_url": build_raw_github_url(self.state, election['start_date'][:4], generated_filename), "ocd_id": ocd_id, "name": name, "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] for election in elections: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] and not election['special'] ] for result in results: meta.append({ "generated_filename": self._standardized_filename(election), "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, year, result['path']), "ocd_id": 'ocd-division/country:us/state:in', "name": 'Indiana', "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] ] for result in results: if result['county']: ocd_id = [ o['ocd_id'] for o in self._jurisdictions() if result['county'] == o['county'] ][0] name = result['county'] else: ocd_id = 'ocd-division/country:us/state:mt' name = "Montana" generated_filename = self._generate_filename( election['start_date'], result) meta.append({ "generated_filename": generated_filename, "raw_url": result['url'], "pre_processed_url": build_raw_github_url(self.state, str(year), result['path']), "ocd_id": ocd_id, "name": name, "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) if year < 2008: for election in elections: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] ] for result in results: generated_filename = self._generate_office_filename( election['direct_links'][0], election['start_date'], election['race_type'], result) meta.append({ "generated_filename": generated_filename, "raw_url": self._build_raw_url(year, result['path']), "pre_processed_url": build_github_url(self.state, generated_filename), "ocd_id": 'ocd-division/country:us/state:wv', "name": 'West Virginia', "election": election['slug'] }) elif year < 2012: for election in elections: meta.append({ "generated_filename": self._generate_statewide_filename(election), "raw_url": election['direct_links'][0], "pre_processed_url": None, "ocd_id": 'ocd-division/country:us/state:wv', "name": 'West Virginia', "election": election['slug'] }) else: for election in elections: csv_links = self._find_csv_links(election['direct_links'][0]) counties = self._jurisdictions() meta.append({ "generated_filename": self._generate_statewide_filename(election), "pre_processed_url": None, "raw_url": election['direct_links'][1], "ocd_id": 'ocd-division/country:us/state:wv', "name": 'West Virginia', "election": election['slug'] }) results = list(zip(counties, csv_links[1:])) for result in results: if election['start_date'] == "2016-11-08" and any( county == result[0]['county'] for county in ['Kanawha', 'Marshall', 'Nicholas', 'Cabell']): meta.append({ "generated_filename": self._generate_county_filename( result[0]['county'], election), "pre_processed_url": build_raw_github_url( self.state, '2016', self._generate_county_filename( result[0]['county'], election)), "raw_url": result[1], "ocd_id": result[0]['ocd_id'], "name": result[0]['county'], "election": election['slug'] }) else: meta.append({ "generated_filename": self._generate_county_filename( result[0]['county'], election), "pre_processed_url": None, "raw_url": result[1], "ocd_id": result[0]['ocd_id'], "name": result[0]['county'], "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) if year > 2008: for election in elections: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] ] for result in results: county = [ c for c in self._jurisdictions() if c['county'] == result['county'] ][0] if year == 2012: generated_filename = self._generate_county_filename( result, election, '.xlsx') else: generated_filename = self._generate_county_filename( result, election, '.xls') meta.append({ "generated_filename": generated_filename, 'raw_url': result['url'], 'raw_extracted_filename': result['raw_extracted_filename'], "ocd_id": county['ocd_id'], "name": county['county'], "election": election['slug'] }) elif year == 2006: for election in elections: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] ] for result in results: county = [ c for c in self._jurisdictions() if c['county'] == result['county'] ][0] generated_filename = self._generate_county_filename( result, election, '.csv') meta.append({ "generated_filename": generated_filename, "pre_processed_url": build_github_url(self.state, generated_filename), "raw_url": result['url'], "ocd_id": county['ocd_id'], "name": county['county'], "election": election['slug'] }) else: for election in elections: results = [ x for x in self._url_paths() if x['date'] == election['start_date'] ] for result in results: county = [ c for c in self._jurisdictions() if c['county'] == result['county'] ][0] if result['special'] and result['raw_extracted_filename']: generated_filename = '20021126__wy__special__general__natrona__state_house__36__precinct.xls' raw_url = build_raw_github_url( self.state, election['start_date'].replace('-', ''), result['raw_extracted_filename']) pre_processed_url = '' elif result['special']: generated_filename = result['path'] raw_url = result['url'] pre_processed_url = '' elif result['raw_extracted_filename'] != '': generated_filename = self._generate_county_filename( result, election, '.' + result['raw_extracted_filename'].split('.')[1]) pre_processed_url = build_raw_github_url( self.state, election['start_date'].replace('-', ''), result['raw_extracted_filename']) raw_url = '' else: generated_filename = self._generate_county_filename( result, election, '.xls') raw_url = build_raw_github_url( self.state, election['start_date'].replace('-', ''), result['raw_extracted_filename']) pre_processed_url = '' meta.append({ "generated_filename": generated_filename, "pre_processed_url": pre_processed_url, "raw_url": raw_url, "ocd_id": county['ocd_id'], "name": county['county'], "election": election['slug'] }) return meta
def _build_metadata(self, year, elections): meta = [] year_int = int(year) if year > 2008: for election in elections: results = [x for x in self._url_paths() if x['date'] == election['start_date']] for result in results: county = [c for c in self._jurisdictions() if c['county'] == result['county']][0] if year > 2010: generated_filename = self._generate_county_filename(result, election, '.xlsx') else: generated_filename = self._generate_county_filename(result, election, '.xls') meta.append({ "generated_filename": generated_filename, 'raw_url': result['url'], 'raw_extracted_filename': result['raw_extracted_filename'], "ocd_id": county['ocd_id'], "name": county['county'], "election": election['slug'] }) elif year == 2006: for election in elections: results = [x for x in self._url_paths() if x['date'] == election['start_date']] for result in results: county = [c for c in self._jurisdictions() if c['county'] == result['county']][0] generated_filename = self._generate_county_filename(result, election, '.csv') meta.append({ "generated_filename": generated_filename, "pre_processed_url": build_github_url(self.state, generated_filename), "raw_url": result['url'], "ocd_id": county['ocd_id'], "name": county['county'], "election": election['slug'] }) else: for election in elections: results = [x for x in self._url_paths() if x['date'] == election['start_date']] for result in results: county = [c for c in self._jurisdictions() if c['county'] == result['county']][0] if result['special'] and result['raw_extracted_filename']: generated_filename = '20021126__wy__special__general__natrona__state_house__36__precinct.xls' raw_url = build_raw_github_url(self.state, election['start_date'].replace('-',''), result['raw_extracted_filename']) pre_processed_url = '' elif result['special']: generated_filename = result['path'] raw_url = result['url'] pre_processed_url = '' elif result['raw_extracted_filename'] != '': generated_filename = self._generate_county_filename(result, election, '.' + result['raw_extracted_filename'].split('.')[1]) pre_processed_url = build_raw_github_url(self.state, election['start_date'].replace('-',''), result['raw_extracted_filename']) raw_url = '' else: generated_filename = self._generate_county_filename(result, election, '.xls') raw_url = build_raw_github_url(self.state, election['start_date'].replace('-',''), result['raw_extracted_filename']) pre_processed_url = '' meta.append({ "generated_filename": generated_filename, "pre_processed_url": pre_processed_url, "raw_url": raw_url, "ocd_id": county['ocd_id'], "name": county['county'], "election": election['slug'] }) return meta