def filter(self, request, qs):
qs = super(RepresentativeListView, self).filter(request, qs)
if 'districts' in request.GET:
qs = qs.filter(boundary__in=request.GET['districts'].split(','))
if 'point' in request.GET:
if app_settings.RESOLVE_POINT_REQUESTS_OVER_HTTP:
url = app_settings.BOUNDARYSERVICE_URL + 'boundaries/?' + urlencode(
{'contains': request.GET['point']})
boundaries = [
boundary_url_to_name(boundary['url']) for boundary in
json.loads(urlopen(url).read().decode())['objects']
]
else:
try:
latitude, longitude = re.sub(
r'[^\d.,-]', '', request.GET['point']).split(',')
wkt = 'POINT(%s %s)' % (longitude, latitude)
boundaries = Boundary.objects.filter(
shape__contains=wkt).values_list('set_id', 'slug')
except ValueError:
raise BadRequest(
"Invalid latitude,longitude '%s' provided." %
request.GET['point'])
boundaries = ['/'.join(boundary) for boundary in boundaries]
qs = qs.filter(boundary__in=boundaries)
return qs
def as_dict(self, sets=None):
r = {
'code': self.code,
'city': self.city,
'province': self.province,
}
if self.centroid:
r['centroid'] = {
'type': 'Point',
'coordinates': [self.centroid.x, self.centroid.y]
}
r.update(self.get_boundaries(sets=sets))
if USE_REPRESENTATIVES:
boundary_names = {}
for match_type in ['concordance', 'centroid']:
for boundary in r.get('boundaries_' + match_type, []):
boundary_names[boundary_url_to_name(
boundary['url'])] = match_type
if boundary_names:
r.update(
self.get_representatives(boundary_names, Representative))
if USE_CANDIDATES:
r.update(
self.get_representatives(boundary_names, Candidate))
return r
def filter(self, request, qs):
qs = super(RepresentativeListView, self).filter(request, qs)
if 'districts' in request.GET:
qs = qs.filter(boundary__in=request.GET['districts'].split(','))
if 'point' in request.GET:
if app_settings.RESOLVE_POINT_REQUESTS_OVER_HTTP:
url = app_settings.BOUNDARYSERVICE_URL + 'boundaries/?' + urlencode({'contains': request.GET['point']})
boundaries = [boundary_url_to_name(boundary['url']) for boundary in json.loads(urlopen(url).read().decode())['objects']]
else:
try:
latitude, longitude = re.sub(r'[^\d.,-]', '', request.GET['point']).split(',')
wkt = 'POINT(%s %s)' % (longitude, latitude)
boundaries = Boundary.objects.filter(shape__contains=wkt).values_list('set_id', 'slug')
except ValueError:
raise BadRequest("Invalid latitude,longitude '%s' provided." % request.GET['point'])
boundaries = ['/'.join(boundary) for boundary in boundaries]
qs = qs.filter(boundary__in=boundaries)
return qs
def as_dict(self, sets=None):
r = {
'code': self.code,
'city': self.city,
'province': self.province,
}
if self.centroid:
r['centroid'] = {
'type': 'Point',
'coordinates': [self.centroid.x, self.centroid.y]
}
r.update(self.get_boundaries(sets=sets))
if USE_REPRESENTATIVES:
boundary_names = {}
for match_type in ['concordance', 'centroid']:
for boundary in r.get('boundaries_' + match_type, []):
boundary_names[boundary_url_to_name(boundary['url'])] = match_type
if boundary_names:
r.update(self.get_representatives(boundary_names, Representative))
if USE_CANDIDATES:
r.update(self.get_representatives(boundary_names, Candidate))
return r
def update_from_data_source(self):
data = json.loads(urlopen(self.data_url).read().decode())
if not (isinstance(data, list) and data): # No data
self.last_import_successful = False
self.save()
return False
# Delete existing data
self.individuals.all().delete()
boundaries = self.get_list_of_boundaries()
boundary_names = dict((
(get_comparison_string(b['name']), b['url']) for b in boundaries
))
boundary_ids = dict((
(b.get('external_id'), b['url']) for b in boundaries
))
url_to_name = dict((
(b['url'], b['name']) for b in boundaries
))
url_to_id = dict((
(b['url'], b.get('external_id')) for b in boundaries
))
for source_rep in data:
rep = self.create_child()
for fieldname in ('name', 'district_name', 'elected_office',
'source_url', 'first_name', 'last_name', 'party_name',
'email', 'url', 'personal_url', 'photo_url', 'district_id',
'gender'):
if source_rep.get(fieldname) is not None:
setattr(rep, fieldname, source_rep[fieldname])
for json_fieldname in ('offices', 'extra'):
if source_rep.get(json_fieldname):
try:
setattr(rep, json_fieldname, json.loads(source_rep.get(json_fieldname)))
except ValueError:
raise Exception("Invalid JSON in %s: %s" % (json_fieldname, source_rep.get(json_fieldname)))
if isinstance(getattr(rep, json_fieldname), list):
for d in getattr(rep, json_fieldname):
if isinstance(d, dict):
for k in d.keys():
if not d[k]:
del d[k]
incumbent = str(source_rep.get('incumbent')).lower()
if incumbent in ('1', 'true', 'yes', 'y'):
rep.incumbent = True
elif incumbent in ('0', 'false', 'no', 'n'):
rep.incumbent = False
if not source_rep.get('name'):
rep.name = ' '.join([component for component in [source_rep.get('first_name'), source_rep.get('last_name')] if component])
if not source_rep.get('first_name') and not source_rep.get('last_name'):
(rep.first_name, rep.last_name) = split_name(rep.name)
boundary_url = None
# Match boundary based on 'boundary_url', then 'district_id', then 'district_name'
if source_rep.get('boundary_url') and _check_boundary_validity(source_rep['boundary_url']):
boundary_url = source_rep['boundary_url']
elif boundaries:
if rep.district_id:
boundary_url = boundary_ids.get(rep.district_id)
if not boundary_url:
boundary_url = boundary_names.get(get_comparison_string(rep.district_name))
if not boundary_url:
logger.warning("%s: Couldn't find district boundary %s in %s" % (self.slug, rep.district_name, self.boundary_set))
else:
rep.boundary = boundary_url_to_name(boundary_url)
if not rep.district_name:
rep.district_name = url_to_name.get(boundary_url, '')
if not rep.district_id:
rep.district_id = url_to_id.get(boundary_url, '')
rep.save()
self.last_import_time = datetime.datetime.now()
self.last_import_successful = True
self.save()
return len(data)
def update_from_data_source(self):
data = json.loads(urlopen(self.data_url).read().decode())
if not (isinstance(data, list) and data): # No data
self.last_import_successful = False
self.save()
return False
# Delete existing data
self.individuals.all().delete()
boundaries = self.get_list_of_boundaries()
boundary_names = dict((
(get_comparison_string(b['name']), b['url']) for b in boundaries
))
boundary_ids = dict((
(b.get('external_id'), b['url']) for b in boundaries
))
url_to_name = dict((
(b['url'], b['name']) for b in boundaries
))
url_to_id = dict((
(b['url'], b.get('external_id')) for b in boundaries
))
for source_rep in data:
rep = self.create_child()
for fieldname in ('name', 'district_name', 'elected_office',
'source_url', 'first_name', 'last_name', 'party_name',
'email', 'url', 'personal_url', 'photo_url', 'district_id',
'gender'):
if source_rep.get(fieldname) is not None:
setattr(rep, fieldname, source_rep[fieldname])
for json_fieldname in ('offices', 'extra'):
if source_rep.get(json_fieldname):
try:
setattr(rep, json_fieldname, json.loads(source_rep.get(json_fieldname)))
except ValueError:
raise Exception("Invalid JSON in %s: %s" % (json_fieldname, source_rep.get(json_fieldname)))
if isinstance(getattr(rep, json_fieldname), list):
for d in getattr(rep, json_fieldname):
if isinstance(d, dict):
for k in d.keys():
if not d[k]:
del d[k]
incumbent = str(source_rep.get('incumbent')).lower()
if incumbent in ('1', 'true', 'yes', 'y'):
rep.incumbent = True
elif incumbent in ('0', 'false', 'no', 'n'):
rep.incumbent = False
if not source_rep.get('name'):
rep.name = ' '.join([component for component in [source_rep.get('first_name'), source_rep.get('last_name')] if component])
if not source_rep.get('first_name') and not source_rep.get('last_name'):
(rep.first_name, rep.last_name) = split_name(rep.name)
boundary_url = None
# Match boundary based on 'boundary_url', then 'district_id', then 'district_name'
if source_rep.get('boundary_url') and _check_boundary_validity(source_rep['boundary_url']):
boundary_url = source_rep['boundary_url']
elif boundaries:
if rep.district_id:
boundary_url = boundary_ids.get(rep.district_id)
if not boundary_url:
boundary_url = boundary_names.get(get_comparison_string(rep.district_name))
if not boundary_url:
logger.warning("%s: Couldn't find district boundary %s in %s" % (self.slug, rep.district_name, self.boundary_set))
else:
rep.boundary = boundary_url_to_name(boundary_url)
if not rep.district_name:
rep.district_name = url_to_name.get(boundary_url, '')
if not rep.district_id:
rep.district_id = url_to_id.get(boundary_url, '')
rep.save()
self.last_import_time = datetime.datetime.now()
self.last_import_successful = True
self.save()
return len(data)
