def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
# each election has a statewide and a county pre-processed file.
statewide_filename = self._generate_statewide_filename(election)
meta.append({
"generated_filename": statewide_filename,
"raw_url": election['direct_links'][0],
"pre_processed_url": build_raw_github_url(self.state, str(year), statewide_filename),
"ocd_id": 'ocd-division/country:us/state:tx',
"name": 'Texas',
"election": election['slug']
})
county_filename = self._generate_county_filename(election)
meta.append({
"generated_filename": county_filename,
"raw_url": election['direct_links'][0],
"pre_processed_url": build_raw_github_url(self.state, str(year), county_filename),
"ocd_id": 'ocd-division/country:us/state:tx',
"name": 'Texas',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
# parish-level file
jurisdiction = 'ocd-division/country:us/state:la'
generated_filename = self._generate_parish_filename(election)
meta.append({
"generated_filename": generated_filename,
"raw_url": election['portal_link'],
"pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename),
"ocd_id": jurisdiction,
"name": 'Louisiana',
"election": election['slug']
})
# precinct-level file
jurisdiction = 'ocd-division/country:us/state:la'
generated_filename = self._generate_precinct_filename(election)
meta.append({
"generated_filename": generated_filename,
"raw_url": election['portal_link'],
"pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename),
"ocd_id": jurisdiction,
"name": 'Louisiana',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if 'special' in election['slug']:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
and x['special'] == True
]
for result in results:
generated_filename = result['path']
if result['county']:
ocd_id = 'ocd-division/country:us/state:mo/county:' + result[
'county'].replace(' ', '_').lower()
else:
ocd_id = 'ocd-division/country:us/state:mo'
meta.append({
"generated_filename":
generated_filename,
"raw_url":
result['url'],
"pre_processed_url":
build_raw_github_url(self.state, str(year),
result['path']),
"ocd_id":
ocd_id,
"name":
'Missouri',
"election":
election['slug']
})
else:
# primary, general and runoff statewide elections have 1 or 2 files per county
# some general runoffs will have smaller numbers of files
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
and x['special'] == False
]
for result in results:
ocd_id = 'ocd-division/country:us/state:mo'
generated_filename = result['path']
meta.append({
"generated_filename":
generated_filename,
"raw_url":
result['url'],
"pre_processed_url":
build_raw_github_url(self.state, str(year),
result['path']),
"ocd_id":
ocd_id,
"name":
'Missouri',
"election":
election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if 'special' in election['slug']:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
for result in results:
generated_filename = result['path']
jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]
ocd_id = jurisdiction['ocd_id']
name = jurisdiction['jurisdiction']
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": build_raw_github_url(self.state, str(year), result['path']),
"ocd_id": ocd_id,
"name": name,
"election": election['slug']
})
else:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] == '']
for result in results:
if result['jurisdiction'] == '':
jurisdiction = 'ocd-division/country:us/state:nv'
generated_filename = self._generate_statewide_filename(result)
pre_processed_url = None
else:
jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]['ocd_id']
generated_filename = result['path']
pre_processed_url = build_raw_github_url(self.state, str(year), result['path'])
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": pre_processed_url,
"ocd_id": jurisdiction,
"name": result['jurisdiction'],
"election": election['slug']
})
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] != '']
for result in results:
jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]['ocd_id']
generated_filename = result['path']
pre_processed_url = build_raw_github_url(self.state, str(year), result['path'])
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": pre_processed_url,
"ocd_id": jurisdiction,
"name": result['jurisdiction'],
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if election['special']:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
else:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False]
for result in results:
if result['url']:
raw_url = result['url']
else:
raw_url = None
if result['county'] == '':
generated_filename = self._generate_filename(election['start_date'], election['race_type'], result)
ocd_id = 'ocd-division/country:us/state:or'
name = "Oregon"
else:
generated_filename = self._generate_county_filename(election['start_date'], result)
ocd_id = 'ocd-division/country:us/state:or/county:%s' % result['county'].lower().replace(" ", "_")
name = result['county']
meta.append({
"generated_filename": generated_filename,
"raw_url": raw_url,
"pre_processed_url": build_raw_github_url(self.state, election['start_date'][0:4], generated_filename),
"ocd_id": ocd_id,
"name": name,
"election": election['slug']
})
# generate precinct files
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if 'special' in election['slug']:
generated_filename = self._generate_filename(election, 'csv')
meta.append({
"generated_filename": generated_filename,
"raw_url": election['direct_links'][0],
"pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename),
"ocd_id": 'ocd-division/country:us/state:ri',
"name": 'Rhode Island',
"election": election['slug']
})
else:
result = [x for x in self._url_paths() if x['date'] == election['start_date']][0]
if result['raw_extracted_filename']:
format = result['raw_extracted_filename'].split('.')[1]
else:
format = 'csv'
meta.append({
"generated_filename": self._generate_filename(election, format),
"raw_url": election['direct_links'][0],
"raw_extracted_filename": result['raw_extracted_filename'],
"pre_processed_url": None,
"ocd_id": 'ocd-division/country:us/state:ri',
"name": 'Rhode Island',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if 'special' in election['slug']:
generated_filename = self._generate_filename(election, 'csv')
meta.append({
"generated_filename": generated_filename,
"raw_url": election['direct_links'][0],
"pre_processed_url": build_raw_github_url(self.state, str(year), generated_filename),
"ocd_id": 'ocd-division/country:us/state:ri',
"name": 'Rhode Island',
"election": election['slug']
})
else:
result = [x for x in self._url_paths() if x['date'] == election['start_date']][0]
if result['raw_extracted_filename']:
format = result['raw_extracted_filename'].split('.')[1]
else:
format = 'csv'
meta.append({
"generated_filename": self._generate_filename(election, format),
"raw_url": election['direct_links'][0],
"raw_extracted_filename": result['raw_extracted_filename'],
"pre_processed_url": None,
"ocd_id": 'ocd-division/country:us/state:ri',
"name": 'Rhode Island',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
if year < 2008:
for election in elections:
results = [x for x in self._url_paths() if x['date'] == election['start_date']]
for result in results:
generated_filename = self._generate_office_filename(election['direct_links'][0], election['start_date'], election['race_type'], result)
meta.append({
"generated_filename": generated_filename,
"raw_url": self._build_raw_url(year, result['path']),
"pre_processed_url": build_github_url(self.state, generated_filename),
"ocd_id": 'ocd-division/country:us/state:wv',
"name": 'West Virginia',
"election": election['slug']
})
elif year < 2012:
for election in elections:
meta.append({
"generated_filename": self._generate_statewide_filename(election),
"raw_url": election['direct_links'][0],
"pre_processed_url": None,
"ocd_id": 'ocd-division/country:us/state:wv',
"name": 'West Virginia',
"election": election['slug']
})
else:
for election in elections:
csv_links = self._find_csv_links(election['direct_links'][0])
counties = self._jurisdictions()
meta.append({
"generated_filename": self._generate_statewide_filename(election),
"pre_processed_url": None,
"raw_url": election['direct_links'][1],
"ocd_id": 'ocd-division/country:us/state:wv',
"name": 'West Virginia',
"election": election['slug']
})
results = list(zip(counties, csv_links[1:]))
for result in results:
if election['start_date'] == "2016-11-08" and any(county == result[0]['county'] for county in ['Kanawha', 'Marshall', 'Nicholas', 'Cabell']):
meta.append({
"generated_filename": self._generate_county_filename(result[0]['county'], election),
"pre_processed_url": build_raw_github_url(self.state, '2016', self._generate_county_filename(result[0]['county'], election)),
"raw_url": result[1],
"ocd_id": result[0]['ocd_id'],
"name": result[0]['county'],
"election": election['slug']
})
else:
meta.append({
"generated_filename": self._generate_county_filename(result[0]['county'], election),
"pre_processed_url": None,
"raw_url": result[1],
"ocd_id": result[0]['ocd_id'],
"name": result[0]['county'],
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if election['special']:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
and x['special'] == True
]
else:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
and x['special'] == False
]
for result in results:
if result['url']:
raw_url = result['url']
else:
raw_url = None
if result['county'] == '':
generated_filename = self._generate_filename(
election['start_date'], election['race_type'], result)
ocd_id = 'ocd-division/country:us/state:or'
name = "Oregon"
else:
generated_filename = self._generate_county_filename(
election['start_date'], result)
ocd_id = 'ocd-division/country:us/state:or/county:%s' % result[
'county'].lower().replace(" ", "_")
name = result['county']
meta.append({
"generated_filename":
generated_filename,
"raw_url":
raw_url,
"pre_processed_url":
build_raw_github_url(self.state,
election['start_date'][0:4],
generated_filename),
"ocd_id":
ocd_id,
"name":
name,
"election":
election['slug']
})
# generate precinct files
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if 'special' in election['slug']:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
for result in results:
jurisdiction = 'ocd-division/country:us/state:nv'
generated_filename = self._generate_statewide_precinct_filename(result)
pre_processed_url = None
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": None,
"ocd_id": jurisdiction,
"name": 'Nevada',
"election": election['slug']
})
else:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] == '']
for result in results:
if result['jurisdiction'] == '':
jurisdiction = 'ocd-division/country:us/state:nv'
generated_filename = self._generate_statewide_filename(result)
pre_processed_url = None
else:
jurisdiction = [c for c in self._jurisdictions() if c['jurisdiction'] == result['jurisdiction']][0]['ocd_id']
generated_filename = result['path']
pre_processed_url = build_raw_github_url(self.state, str(year), result['path'])
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": pre_processed_url,
"ocd_id": jurisdiction,
"name": result['jurisdiction'],
"election": election['slug']
})
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False and x['precinct'] != '']
for result in results:
jurisdiction = 'ocd-division/country:us/state:nv'
generated_filename = self._generate_statewide_precinct_filename(result)
pre_processed_url = None
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": pre_processed_url,
"ocd_id": jurisdiction,
"name": 'Nevada',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if 'special' in election['slug']:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
for result in results:
generated_filename = result['path']
if result['county']:
ocd_id = 'ocd-division/country:us/state:mo/county:' + result['county'].replace(' ','_').lower()
else:
ocd_id = 'ocd-division/country:us/state:mo'
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": build_raw_github_url(self.state, str(year), result['path']),
"ocd_id": ocd_id,
"name": 'Missouri',
"election": election['slug']
})
else:
# primary, general and runoff statewide elections have 1 or 2 files per county
# some general runoffs will have smaller numbers of files
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False]
for result in results:
ocd_id = 'ocd-division/country:us/state:mo'
generated_filename = result['path']
meta.append({
"generated_filename": generated_filename,
"raw_url": result['url'],
"pre_processed_url": build_raw_github_url(self.state, str(year), result['path']),
"ocd_id": ocd_id,
"name": 'Missouri',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
for election in elections:
results = [x for x in self._url_paths()
if x['date'] == election['start_date'] and
not election['special']]
for result in results:
meta.append({
"generated_filename": self._standardized_filename(election),
"raw_url": result['url'],
"pre_processed_url": build_raw_github_url(
self.state, year, result['path']),
"ocd_id": 'ocd-division/country:us/state:in',
"name": 'Indiana',
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
try:
raw_url = election['direct_links'][0] # In reality, the election may have multiple source files, but we shouldn't be using the raw_url for anything
except IndexError:
raw_url = election['direct_link']
generated_filename = self._generate_filename(election)
ocd_id = 'ocd-division/country:us/state:wi'
name = "Wisconsin"
meta.append({
"generated_filename": generated_filename,
"raw_url": raw_url,
"pre_processed_url": build_raw_github_url(self.state, election['start_date'][:4], generated_filename),
"ocd_id": ocd_id,
"name": name,
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
for election in elections:
results = [
x for x in self._url_paths() if
x['date'] == election['start_date'] and not election['special']
]
for result in results:
meta.append({
"generated_filename":
self._standardized_filename(election),
"raw_url":
result['url'],
"pre_processed_url":
build_raw_github_url(self.state, year, result['path']),
"ocd_id":
'ocd-division/country:us/state:in',
"name":
'Indiana',
"election":
election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
]
for result in results:
if result['county']:
ocd_id = [
o['ocd_id'] for o in self._jurisdictions()
if result['county'] == o['county']
][0]
name = result['county']
else:
ocd_id = 'ocd-division/country:us/state:mt'
name = "Montana"
generated_filename = self._generate_filename(
election['start_date'], result)
meta.append({
"generated_filename":
generated_filename,
"raw_url":
result['url'],
"pre_processed_url":
build_raw_github_url(self.state, str(year),
result['path']),
"ocd_id":
ocd_id,
"name":
name,
"election":
election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
if year < 2008:
for election in elections:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
]
for result in results:
generated_filename = self._generate_office_filename(
election['direct_links'][0], election['start_date'],
election['race_type'], result)
meta.append({
"generated_filename":
generated_filename,
"raw_url":
self._build_raw_url(year, result['path']),
"pre_processed_url":
build_github_url(self.state, generated_filename),
"ocd_id":
'ocd-division/country:us/state:wv',
"name":
'West Virginia',
"election":
election['slug']
})
elif year < 2012:
for election in elections:
meta.append({
"generated_filename":
self._generate_statewide_filename(election),
"raw_url":
election['direct_links'][0],
"pre_processed_url":
None,
"ocd_id":
'ocd-division/country:us/state:wv',
"name":
'West Virginia',
"election":
election['slug']
})
else:
for election in elections:
csv_links = self._find_csv_links(election['direct_links'][0])
counties = self._jurisdictions()
meta.append({
"generated_filename":
self._generate_statewide_filename(election),
"pre_processed_url":
None,
"raw_url":
election['direct_links'][1],
"ocd_id":
'ocd-division/country:us/state:wv',
"name":
'West Virginia',
"election":
election['slug']
})
results = list(zip(counties, csv_links[1:]))
for result in results:
if election['start_date'] == "2016-11-08" and any(
county == result[0]['county'] for county in
['Kanawha', 'Marshall', 'Nicholas', 'Cabell']):
meta.append({
"generated_filename":
self._generate_county_filename(
result[0]['county'], election),
"pre_processed_url":
build_raw_github_url(
self.state, '2016',
self._generate_county_filename(
result[0]['county'], election)),
"raw_url":
result[1],
"ocd_id":
result[0]['ocd_id'],
"name":
result[0]['county'],
"election":
election['slug']
})
else:
meta.append({
"generated_filename":
self._generate_county_filename(
result[0]['county'], election),
"pre_processed_url":
None,
"raw_url":
result[1],
"ocd_id":
result[0]['ocd_id'],
"name":
result[0]['county'],
"election":
election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
if year > 2008:
for election in elections:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
]
for result in results:
county = [
c for c in self._jurisdictions()
if c['county'] == result['county']
][0]
if year == 2012:
generated_filename = self._generate_county_filename(
result, election, '.xlsx')
else:
generated_filename = self._generate_county_filename(
result, election, '.xls')
meta.append({
"generated_filename":
generated_filename,
'raw_url':
result['url'],
'raw_extracted_filename':
result['raw_extracted_filename'],
"ocd_id":
county['ocd_id'],
"name":
county['county'],
"election":
election['slug']
})
elif year == 2006:
for election in elections:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
]
for result in results:
county = [
c for c in self._jurisdictions()
if c['county'] == result['county']
][0]
generated_filename = self._generate_county_filename(
result, election, '.csv')
meta.append({
"generated_filename":
generated_filename,
"pre_processed_url":
build_github_url(self.state, generated_filename),
"raw_url":
result['url'],
"ocd_id":
county['ocd_id'],
"name":
county['county'],
"election":
election['slug']
})
else:
for election in elections:
results = [
x for x in self._url_paths()
if x['date'] == election['start_date']
]
for result in results:
county = [
c for c in self._jurisdictions()
if c['county'] == result['county']
][0]
if result['special'] and result['raw_extracted_filename']:
generated_filename = '20021126__wy__special__general__natrona__state_house__36__precinct.xls'
raw_url = build_raw_github_url(
self.state,
election['start_date'].replace('-', ''),
result['raw_extracted_filename'])
pre_processed_url = ''
elif result['special']:
generated_filename = result['path']
raw_url = result['url']
pre_processed_url = ''
elif result['raw_extracted_filename'] != '':
generated_filename = self._generate_county_filename(
result, election, '.' +
result['raw_extracted_filename'].split('.')[1])
pre_processed_url = build_raw_github_url(
self.state,
election['start_date'].replace('-', ''),
result['raw_extracted_filename'])
raw_url = ''
else:
generated_filename = self._generate_county_filename(
result, election, '.xls')
raw_url = build_raw_github_url(
self.state,
election['start_date'].replace('-', ''),
result['raw_extracted_filename'])
pre_processed_url = ''
meta.append({
"generated_filename": generated_filename,
"pre_processed_url": pre_processed_url,
"raw_url": raw_url,
"ocd_id": county['ocd_id'],
"name": county['county'],
"election": election['slug']
})
return meta
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
if year > 2008:
for election in elections:
results = [x for x in self._url_paths() if x['date'] == election['start_date']]
for result in results:
county = [c for c in self._jurisdictions() if c['county'] == result['county']][0]
if year > 2010:
generated_filename = self._generate_county_filename(result, election, '.xlsx')
else:
generated_filename = self._generate_county_filename(result, election, '.xls')
meta.append({
"generated_filename": generated_filename,
'raw_url': result['url'],
'raw_extracted_filename': result['raw_extracted_filename'],
"ocd_id": county['ocd_id'],
"name": county['county'],
"election": election['slug']
})
elif year == 2006:
for election in elections:
results = [x for x in self._url_paths() if x['date'] == election['start_date']]
for result in results:
county = [c for c in self._jurisdictions() if c['county'] == result['county']][0]
generated_filename = self._generate_county_filename(result, election, '.csv')
meta.append({
"generated_filename": generated_filename,
"pre_processed_url": build_github_url(self.state, generated_filename),
"raw_url": result['url'],
"ocd_id": county['ocd_id'],
"name": county['county'],
"election": election['slug']
})
else:
for election in elections:
results = [x for x in self._url_paths() if x['date'] == election['start_date']]
for result in results:
county = [c for c in self._jurisdictions() if c['county'] == result['county']][0]
if result['special'] and result['raw_extracted_filename']:
generated_filename = '20021126__wy__special__general__natrona__state_house__36__precinct.xls'
raw_url = build_raw_github_url(self.state, election['start_date'].replace('-',''), result['raw_extracted_filename'])
pre_processed_url = ''
elif result['special']:
generated_filename = result['path']
raw_url = result['url']
pre_processed_url = ''
elif result['raw_extracted_filename'] != '':
generated_filename = self._generate_county_filename(result, election, '.' + result['raw_extracted_filename'].split('.')[1])
pre_processed_url = build_raw_github_url(self.state, election['start_date'].replace('-',''), result['raw_extracted_filename'])
raw_url = ''
else:
generated_filename = self._generate_county_filename(result, election, '.xls')
raw_url = build_raw_github_url(self.state, election['start_date'].replace('-',''), result['raw_extracted_filename'])
pre_processed_url = ''
meta.append({
"generated_filename": generated_filename,
"pre_processed_url": pre_processed_url,
"raw_url": raw_url,
"ocd_id": county['ocd_id'],
"name": county['county'],
"election": election['slug']
})
return meta