def fake_requests_for_mapit(url):
"""Return reduced MapIt output for some known URLs"""
if url == urljoin(settings.MAPIT_BASE_URL, '/postcode/sw1a1aa'):
status_code = 200
json_result = sw1a1aa_result
elif url == urljoin(settings.MAPIT_BASE_URL, '/postcode/se240ag'):
status_code = 200
json_result = se240ag_result
elif url == urljoin(settings.MAPIT_BASE_URL, '/postcode/cb28rq'):
status_code = 404
json_result = {
"code": 404,
"error": "No Postcode matches the given query."
}
elif url == urljoin(settings.MAPIT_BASE_URL, '/postcode/foobar'):
status_code = 400
json_result = {
"code": 400,
"error": "Postcode 'FOOBAR' is not valid."
}
else:
raise Exception("URL that hasn't been mocked yet: " + url)
return Mock(**{
'json.return_value': json_result,
'status_code': status_code
})
def test_delete_elections_with_matches(self, mock_requests):
# import some data
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(EE_BASE_URL, "/api/elections/?current=True"):
local_highland,
urljoin(EE_BASE_URL, "/api/elections/?current=1&deleted=1"):
no_results,
})
call_command("uk_create_elections_from_every_election")
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 1)
self.assertEqual(every_election.YNRElection.objects.all().count(), 1)
# now we've switched the fixtures round
# so the records we just imported are deleted in EE
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(EE_BASE_URL, "/api/elections/?current=True"):
no_results,
urljoin(EE_BASE_URL, "/api/elections/?current=1&deleted=1"):
local_highland,
})
call_command("uk_create_elections_from_every_election")
# we should delete the records we just imported
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 0)
self.assertEqual(every_election.YNRElection.objects.all().count(), 0)
def fake_requests_for_mapit(url):
"""Return reduced MapIt output for some known URLs"""
if url == urljoin(settings.MAPIT_BASE_URL, '/postcode/sw1a1aa'):
status_code = 200
json_result = sw1a1aa_result
elif url == urljoin(settings.MAPIT_BASE_URL, '/postcode/se240ag'):
status_code = 200
json_result = se240ag_result
elif url == urljoin(settings.MAPIT_BASE_URL, '/postcode/cb28rq'):
status_code = 404
json_result = {
"code": 404,
"error": "No Postcode matches the given query."
}
elif url == urljoin(settings.MAPIT_BASE_URL, '/postcode/foobar'):
status_code = 400
json_result = {
"code": 400,
"error": "Postcode 'FOOBAR' is not valid."
}
else:
raise Exception("URL that hasn't been mocked yet: " + url)
return Mock(**{
'json.return_value': json_result,
'status_code': status_code
})
def test_delete_elections_invalid_input_insert_and_delete(
self, mock_requests):
# import some data
# just so we've got a non-empty DB
mock_requests.get.side_effect = (
fake_requests_each_type_of_election_on_one_day)
call_command("uk_create_elections_from_every_election")
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 15)
self.assertEqual(every_election.YNRElection.objects.all().count(), 10)
# this simulates a situation where EE reports
# the same election/s as deleted and not deleted
# this makes no sense and shouldn't happen but
# if it does we should not delete anything
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(EE_BASE_URL, "/api/elections/?current=True"):
local_highland,
urljoin(EE_BASE_URL, "/api/elections/?current=1&deleted=1"):
local_highland,
})
# make sure we throw an exception
with self.assertRaises(Exception):
call_command("uk_create_elections_from_every_election")
# we should also roll back the whole transaction
# so nothing is inserted or deleted
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 15)
self.assertEqual(every_election.YNRElection.objects.all().count(), 10)
def test_delete_elections_with_related_membership(self, mock_requests):
# import some data
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(EE_BASE_URL, "/api/elections/?current=True"):
local_highland,
urljoin(EE_BASE_URL, "/api/elections/?current=1&deleted=1"):
no_results,
})
call_command("uk_create_elections_from_every_election")
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 1)
self.assertEqual(every_election.YNRElection.objects.all().count(), 1)
# create a membership which references the PEE we just imported
MembershipFactory(
person=PersonFactory.create(id=2009, name="Tessa Jowell"),
post_election=every_election.PostExtraElection.objects.all()[0],
)
# now we've switched the fixtures round
# so the records we just imported are deleted in EE
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(EE_BASE_URL, "/api/elections/?current=True"):
no_results,
urljoin(EE_BASE_URL, "/api/elections/?current=1&deleted=1"):
local_highland,
})
# make sure we throw an exception
with self.assertRaises(Exception):
call_command("uk_create_elections_from_every_election")
# we should also roll back the whole transaction so nothing is deleted
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 1)
self.assertEqual(every_election.YNRElection.objects.all().count(), 1)
def test_delete_elections_no_matches(self, mock_requests):
# import some data
# just so we've got a non-empty DB
mock_requests.get.side_effect = (
fake_requests_each_type_of_election_on_one_day)
call_command("uk_create_elections_from_every_election")
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 15)
self.assertEqual(every_election.YNRElection.objects.all().count(), 10)
# now we're going to delete some stuff
# but none of the elections in the
# local_highland fixture
# match anything we just imported
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(EE_BASE_URL, "/api/elections/?current=True"):
no_results,
urljoin(EE_BASE_URL, "/api/elections/?current=1&deleted=1"):
local_highland,
})
# this should finish cleanly without complaining
call_command("uk_create_elections_from_every_election")
# nothing in our DB should have changed
self.assertEqual(
every_election.PostExtraElection.objects.all().count(), 15)
self.assertEqual(every_election.YNRElection.objects.all().count(), 10)
def fake_requests_for_every_election(url, *args, **kwargs):
"""Return reduced EE output for some known URLs"""
EE_BASE_URL = getattr(
settings, "EE_BASE_URL", "https://elections.democracyclub.org.uk/")
if url == urljoin(EE_BASE_URL,
'/api/elections/?postcode=se240ag'):
status_code = 200
json_result = ee_se240ag_result
elif url == urljoin(EE_BASE_URL,
'/api/elections/?postcode=sw1a1aa'):
status_code = 200
json_result = ee_sw1a1aa_result
elif url == urljoin(EE_BASE_URL, '/api/elections/?postcode=cb28rq'):
status_code = 404
json_result = {
"code": 404,
"error": "No Postcode matches the given query."
}
else:
raise Exception("URL that hasn't been mocked yet: " + url)
return Mock(**{
'json.return_value': json_result,
'status_code': status_code
})
def fake_requests_for_every_election(url, *args, **kwargs):
"""Return reduced EE output for some known URLs"""
EE_BASE_URL = getattr(settings, "EE_BASE_URL",
"https://elections.democracyclub.org.uk/")
if url == urljoin(EE_BASE_URL, '/api/elections/?postcode=se240ag'):
status_code = 200
json_result = ee_se240ag_result
elif url == urljoin(EE_BASE_URL, '/api/elections/?postcode=se240xx'):
status_code = 400
json_result = {"detail": "Unknown postcode"}
elif url == urljoin(EE_BASE_URL,
'/api/elections/?coords=-0.143207%2C51.5'):
status_code = 200
json_result = ee_se240ag_result
elif url == urljoin(EE_BASE_URL, '/api/elections/?postcode=sw1a1aa'):
status_code = 200
json_result = ee_sw1a1aa_result
elif url == urljoin(EE_BASE_URL, '/api/elections/?postcode=cb28rq'):
status_code = 404
json_result = {
"code": 404,
"error": 'The url "{}" couldn’t be found'.format(url)
}
else:
raise Exception("URL that hasn't been mocked yet: " + url)
return Mock(**{
'json.return_value': json_result,
'status_code': status_code
})
def test_import_post_duplicate_slugs(self, mock_requests):
"""
Regression test to check that a post with duplicate names
:param mock_requests:
:return:
"""
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(
EE_BASE_URL,
"/api/elections/?poll_open_date__gte=2018-01-03",
):
duplicate_post_names,
urljoin(
EE_BASE_URL,
"/api/elections/?deleted=1&poll_open_date__gte=2018-01-03",
):
local_highland,
})
call_command("uk_create_elections_from_every_election")
post_a, post_b = Post.objects.all().order_by("-slug",
"-organization__name")
self.assertEqual(post_a.slug, "st-michaels")
self.assertEqual(post_a.identifier, "SUR:st-michaels")
self.assertEqual(post_a.start_date, "2019-05-02")
self.assertEqual(post_a.organization.name,
"Surrey Heath Borough Council")
self.assertEqual(post_b.slug, "st-michaels")
self.assertEqual(post_b.identifier, "ALL:st-michaels")
self.assertEqual(post_a.start_date, "2019-05-02")
self.assertEqual(post_b.organization.name, "Allerdale Borough Council")
def fake_requests_for_every_election(url, *args, **kwargs):
"""Return reduced EE output for some known URLs"""
EE_BASE_URL = getattr(settings, "EE_BASE_URL",
"https://elections.democracyclub.org.uk/")
result = None
if url == urljoin(EE_BASE_URL, 'api/elections/?current=True'):
status_code = 200
result = current_elections
if url == urljoin(
EE_BASE_URL,
'/api/elections/?current=True&limit=100&offset=100'): # noqa
status_code = 200
result = current_elections_page_2
if url == urljoin(
EE_BASE_URL,
'/api/elections/?current=True&poll_open_date=2019-01-17'): # noqa
status_code = 200
result = each_type_of_election_on_one_day
if not result:
raise Exception("URL that hasn't been mocked yet: " + url)
return Mock(**{'json.return_value': result, 'status_code': status_code})
def test_delete_elections_invalid_input_non_empty_election(
self, mock_requests):
# import some data
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(
EE_BASE_URL,
"/api/elections/?poll_open_date__gte=2018-01-03",
):
local_highland,
urljoin(
EE_BASE_URL,
"/api/elections/?deleted=1&poll_open_date__gte=2018-01-03",
):
no_results,
})
call_command("uk_create_elections_from_every_election")
self.assertEqual(every_election.Ballot.objects.all().count(), 1)
self.assertEqual(every_election.YNRElection.objects.all().count(), 1)
# this simulates a situation where EE reports
# a ballot as current but its parent election as
# deleted this makes no sense and shouldn't happen
# but if it does we should not delete anything
current_elections = {
"count": 1,
"next": None,
"previous": None,
"results": [local_highland["results"][1]],
}
deleted_elections = {
"count": 1,
"next": None,
"previous": None,
"results": [local_highland["results"][0]],
}
mock_requests.get.side_effect = create_mock_with_fixtures({
urljoin(
EE_BASE_URL,
"/api/elections/?poll_open_date__gte=2018-01-03",
):
current_elections,
urljoin(
EE_BASE_URL,
"/api/elections/?deleted=1&poll_open_date__gte=2018-01-03",
):
deleted_elections,
})
# make sure we throw an exception
with self.assertRaises(Exception):
call_command("uk_create_elections_from_every_election")
# we should also roll back the whole transaction
# so nothing is inserted or deleted
self.assertEqual(every_election.Ballot.objects.all().count(), 1)
self.assertEqual(every_election.YNRElection.objects.all().count(), 1)
def get_areas_from_postcode(postcode):
cache_key = 'mapit-postcode:' + postcode
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL, '/postcode/' + urlquote(postcode))
r = requests.get(url, headers={'User-Agent': 'scraper/sym', })
if r.status_code == 200:
mapit_result = r.json()
areas = get_known_area_types(mapit_result['areas'])
cache.set(cache_key, areas, settings.MAPIT_CACHE_SECONDS)
return areas
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(
postcode
)
)
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
postcode
)
)
def get_logout_url(self, redirect_url=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(self.server_url, 'logout')
if redirect_url:
param_name = self._get_logout_redirect_parameter_name()
url += '?' + urllib_parse.urlencode({param_name: redirect_url})
return url
def verify_ticket(self, ticket):
"""Verifies CAS 2.0+ XML-based authentication ticket."""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
user = None
pgtiou = None
params = [('ticket', ticket), ('service', self.service_url)]
if self.proxy_callback:
params.append(('pgtUrl', self.proxy_callback))
url = (urllib_parse.urljoin(self.server_url, 'serviceValidate') + '?' +
urllib_parse.urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('authenticationSuccess'):
for element in tree[0]:
if element.tag.endswith('user'):
user = element.text
elif element.tag.endswith('proxyGrantingTicket'):
pgtiou = element.text
return user, None, pgtiou
else:
return None, None, None
finally:
page.close()
def _verify_cas3(ticket, service):
"""Verifies CAS 3.0+ XML-based authentication ticket and returns extended attributes.
Returns username on success and None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
params = [('ticket', ticket), ('service', service)]
url = (urllib_parse.urljoin(settings.CAS_SERVER_URL, 'proxyValidate') + '?' +
urllib_parse.urlencode(params))
page = urlopen(url)
try:
user = None
attributes = {}
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('authenticationSuccess'):
for element in tree[0]:
if element.tag.endswith('user'):
user = element.text
elif element.tag.endswith('attributes'):
for attribute in element:
attributes[attribute.tag.split("}").pop()] = attribute.text
return user, attributes
finally:
page.close()
def load_data(self, code):
data = requests.get(
urljoin(
self.mapit_base_url,
"areas/{code}".format(mapit_base_url=self.mapit_base_url,
code=code))).json()
self.update(data)
def format_elections_html_url(self):
endpoint = "mgElectionElectionAreaResults.aspx"
url = urljoin(self.base_domain, endpoint)
url = "https://{}?Page=all&EID=".format(url)
if self.http_only:
url = url.replace("https://", "http://")
return url
def get_logout_url(self, redirect_url=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(self.server_url, 'logout')
if redirect_url:
param_name = self._get_logout_redirect_parameter_name()
url += '?' + urllib_parse.urlencode({param_name: redirect_url})
return url
def upload_images(self, emblems, party_extra):
content_type = ContentType.objects.get_for_model(party_extra)
sort_emblems(emblems, party_extra.slug)
primary = True
for emblem in emblems:
emblem_id = str(emblem['Id'])
ntf = NamedTemporaryFile(delete=False)
image_url = urljoin(base_emblem_url, emblem_id)
r = requests.get(image_url)
with open(ntf.name, 'w') as f:
f.write(r.content)
mime_type = self.mime_type_magic.from_file(ntf.name)
extension = mimetypes.guess_extension(mime_type)
leafname = 'Emblem_{0}{1}'.format(emblem_id, extension)
desired_storage_path = join('images', leafname)
fname = join(emblem_directory, leafname)
move(ntf.name, fname)
md5sum = get_file_md5sum(fname)
ImageExtra.objects.update_or_create_from_file(
fname,
desired_storage_path,
md5sum=md5sum,
defaults={
'uploading_user':None,
'md5sum': md5sum,
'notes': emblem['MonochromeDescription'],
'base__source': 'The Electoral Commission',
'base__is_primary': primary,
'base__object_id': party_extra.id,
'base__content_type_id': content_type.id,
}
)
primary = False
def get_areas_from_postcode(postcode):
cache_key = 'mapit-postcode:' + postcode
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL, '/postcode/' + urlquote(postcode))
r = requests.get(url)
if r.status_code == 200:
mapit_result = r.json()
areas = get_known_area_types(mapit_result['areas'])
cache.set(cache_key, areas, settings.MAPIT_CACHE_SECONDS)
return areas
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(
postcode
)
)
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
postcode
)
)
def get_areas_from_coords(coords):
url = urljoin(
settings.MAPIT_BASE_URL,
'/point/4326/' + urlquote(coords)
)
cache_key = 'mapit-postcode:' + coords
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
r = requests.get(url, headers={'User-Agent': 'scraper/sym', })
if r.status_code == 200:
mapit_result = r.json()
areas = get_known_area_types(mapit_result)
cache.set(url, areas, settings.MAPIT_CACHE_SECONDS)
return areas
elif r.status_code == 400:
mapit_result = r.json()
raise BadCoordinatesException(mapit_result['error'])
elif r.status_code == 404:
raise BadCoordinatesException(
'The coordinates "{0}" could not be found'.format(
coords
)
)
else:
raise UnknownMapitException(
'Unknown MapIt error for coordinates "{0}"'.format(
coords
)
)
def get_areas_from_postcode(original_postcode):
postcode = re.sub(r'(?ms)\s*', '', original_postcode.lower())
if re.search(r'[^a-z0-9]', postcode):
raise BadPostcodeException(
_('There were disallowed characters in "{0}"').format(
original_postcode))
cache_key = 'mapit-postcode:' + postcode
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL,
'/postcode/{0}'.format(urlquote(postcode)))
r = requests.get(url)
if r.status_code == 200:
mapit_result = r.json()
result = get_known_area_types(mapit_result['areas'])
cache.set(cache_key, result, settings.MAPIT_CACHE_SECONDS)
return result
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(
original_postcode))
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
original_postcode))
def get_areas_from_postcode(original_postcode):
postcode = re.sub(r'(?ms)\s*', '', original_postcode.lower())
if re.search(r'[^a-z0-9]', postcode):
raise BadPostcodeException(
_('There were disallowed characters in "{0}"').format(original_postcode)
)
cache_key = 'mapit-postcode:' + postcode
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL,
'/postcode/{0}'.format(urlquote(postcode)))
r = requests.get(url, headers={'User-Agent': 'scraper/sym', })
if r.status_code == 200:
mapit_result = r.json()
result = get_known_area_types(mapit_result['areas'])
cache.set(cache_key, result, settings.MAPIT_CACHE_SECONDS)
return result
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(original_postcode)
)
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
original_postcode
)
)
def upload_images(self, emblems, party_extra):
content_type = ContentType.objects.get_for_model(party_extra)
sort_emblems(emblems, party_extra.slug)
primary = True
for emblem in emblems:
emblem_id = str(emblem['Id'])
ntf = NamedTemporaryFile(delete=False)
image_url = urljoin(base_emblem_url, emblem_id)
r = requests.get(image_url)
with open(ntf.name, 'w') as f:
f.write(r.content)
mime_type = self.mime_type_magic.from_file(ntf.name)
extension = mimetypes.guess_extension(mime_type)
leafname = 'Emblem_{0}{1}'.format(emblem_id, extension)
desired_storage_path = join('images', leafname)
fname = join(emblem_directory, leafname)
move(ntf.name, fname)
md5sum = get_file_md5sum(fname)
ImageExtra.objects.update_or_create_from_file(
fname,
desired_storage_path,
md5sum=md5sum,
base__object_id=party_extra.id,
base__content_type_id=content_type.id,
defaults={
'uploading_user': None,
'notes': emblem['MonochromeDescription'],
'base__source': 'The Electoral Commission',
'base__is_primary': primary,
})
primary = False
def get_areas_from_coords(coords):
url = urljoin(
settings.MAPIT_BASE_URL,
'/point/4326/' + urlquote(coords)
)
cache_key = 'mapit-postcode:' + coords
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
r = requests.get(url)
if r.status_code == 200:
mapit_result = r.json()
areas = get_known_area_types(mapit_result)
cache.set(url, areas, settings.MAPIT_CACHE_SECONDS)
return areas
elif r.status_code == 400:
mapit_result = r.json()
raise BadCoordinatesException(mapit_result['error'])
elif r.status_code == 404:
raise BadCoordinatesException(
'The coordinates "{0}" could not be found'.format(
coords
)
)
else:
raise UnknownMapitException(
'Unknown MapIt error for coordinates "{0}"'.format(
coords
)
)
def verify_ticket(self, ticket):
"""Verifies CAS 2.0+ XML-based authentication ticket.
Returns username on success and None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
user = None
pgtiou = None
params = [('ticket', ticket), ('service', self.service_url)]
if self.proxy_callback:
params.append(('pgtUrl', self.proxy_callback))
url = (urllib_parse.urljoin(self.server_url, 'serviceValidate') + '?' +
urllib_parse.urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('authenticationSuccess'):
for element in tree[0]:
if element.tag.endswith('user'):
user = element.text
elif element.tag.endswith('proxyGrantingTicket'):
pgtiou = element.text
return user, None, pgtiou
else:
return None, None, None
finally:
page.close()
def get_wmc_from_postcode(original_postcode):
postcode = re.sub(r'(?ms)\s*', '', original_postcode.lower())
if re.search(r'[^a-z0-9]', postcode):
raise BadPostcodeException(
_('There were disallowed characters in "{0}"').format(
original_postcode))
cached_result = cache.get(postcode)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL,
'/postcode/{0}'.format(urlquote(postcode)))
r = requests.get(url)
if r.status_code == 200:
mapit_result = r.json()
wmc = mapit_result.get('shortcuts', {}).get('WMC')
if not wmc:
raise NoConstituencyForPostcodeException(
_('No constituency found for the postcode "{0}"').format(
original_postcode))
cache.set(postcode, wmc, settings.MAPIT_CACHE_SECONDS)
return wmc
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(
original_postcode))
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
original_postcode))
def format_elections_index_url(self):
endpoint = "mgManageElectionResults.aspx"
url = urljoin(self.base_domain, endpoint)
url = "https://{}".format(url)
if self.http_only:
url = url.replace("https://", "http://")
return url
def get_verification_response(self, ticket):
params = [('ticket', ticket), ('service', self.service_url)]
if self.proxy_callback:
params.append(('pgtUrl', self.proxy_callback))
base_url = urllib_parse.urljoin(self.server_url, 'proxyValidate')
url = base_url + '?' + urllib_parse.urlencode(params)
page = urlopen(url)
return page.read()
def get_verification_response(self, ticket):
params = [('ticket', ticket), ('service', self.service_url)]
if self.proxy_callback:
params.append(('pgtUrl', self.proxy_callback))
base_url = urllib_parse.urljoin(self.server_url, 'proxyValidate')
url = base_url + '?' + urllib_parse.urlencode(params)
page = urlopen(url)
return page.read()
def __make_absolute__(self, path):
"""
When `relative_paths` is True, fully qualify path with API Path.
"""
if path:
if SWAGGER_SETTINGS.get('relative_paths', False):
path = urljoin(SWAGGER_SETTINGS.get('api_path'), path)
return path
def get_areas_from_postcode(original_postcode):
postcode = re.sub(r'(?ms)\s*', '', original_postcode.lower())
if re.search(r'[^a-z0-9]', postcode):
raise BadPostcodeException(
_('There were disallowed characters in "{0}"').format(original_postcode)
)
cache_key = 'mapit-postcode:' + postcode
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL,
'/postcode/{0}'.format(urlquote(postcode)))
r = requests.get(url)
if r.status_code == 200:
mapit_result = r.json()
result = []
for mapit_area in mapit_result['areas'].values():
areas = Area.objects.filter(
extra__type__name=mapit_area['type'],
identifier=format_code_from_area(mapit_area)
)
if areas.exists():
is_no_data_area = False
for area in areas:
for child_area in area.children.all():
if child_area.identifier.startswith('NODATA:'):
is_no_data_area = True
break
if is_no_data_area:
area_type = "NODATA"
else:
area_type = mapit_area['type']
result.append((
area_type,
format_code_from_area(mapit_area))
)
result = sorted(result, key=area_sort_key)
cache.set(cache_key, result, settings.MAPIT_CACHE_SECONDS)
return result
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(original_postcode)
)
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
original_postcode
)
)
def _logout_url(request, next_page=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(settings.CAS_SERVER_URL, 'logout')
if next_page:
protocol = ('http://', 'https://')[request.is_secure()]
host = request.get_host()
url += '?' + urllib_parse.urlencode({'url': protocol + host + next_page})
return url
def _login_url(service):
"""Generates CAS login URL"""
params = {'service': service}
if settings.CAS_RENEW:
params.update({'renew': 'true'})
if settings.CAS_EXTRA_LOGIN_PARAMS:
params.update(settings.CAS_EXTRA_LOGIN_PARAMS)
return urllib_parse.urljoin(settings.CAS_SERVER_URL, 'login') + '?' + urllib_parse.urlencode(params)
def __make_absolute__(self, path):
"""
When `relative_paths` is True, fully qualify path with API Path.
"""
if path:
if SWAGGER_SETTINGS.get('relative_paths', False):
path = urljoin(SWAGGER_SETTINGS.get('api_path'), path)
return path
def check_address(address_string, country=None):
tidied_address_before_country = address_string.strip()
if country is None:
tidied_address = tidied_address_before_country
else:
tidied_address = tidied_address_before_country + ', ' + country
try:
location_results = Geocoder.geocode(tidied_address)
except GeocoderError:
message = _("Failed to find a location for '{0}'")
raise ValidationError(message.format(tidied_address_before_country))
lat, lon = location_results[0].coordinates
all_mapit_json = []
queries_to_try = defaultdict(set)
for election in Election.objects.current().prefetch_related('area_types'):
area_types = election.area_types.values_list('name', flat=True)
queries_to_try[election.area_generation].update(area_types)
for area_generation, area_types in queries_to_try.items():
mapit_lookup_url = urljoin(settings.MAPIT_BASE_URL,
'point/4326/{lon},{lat}'.format(
lon=lon,
lat=lat,
))
mapit_lookup_url += '?type=' + ','.join(area_types)
mapit_lookup_url += '&generation={0}'.format(election.area_generation)
mapit_result = requests.get(mapit_lookup_url)
mapit_json = mapit_result.json()
if 'error' in mapit_json:
message = _("The area lookup returned an error: '{error}'")
raise ValidationError(message.format(error=mapit_json['error']))
all_mapit_json += mapit_json.items()
sorted_mapit_results = sorted(
all_mapit_json,
key=lambda t: (t[1]['type'], int(t[0]))
)
if not sorted_mapit_results:
message = _("The address '{0}' appears to be outside the area this site knows about")
raise ValidationError(message.format(tidied_address_before_country))
types_and_areas = [
{
'area_type_code': a[1]['type'],
'area_id': a[0],
}
for a in sorted_mapit_results
]
if settings.AREAS_TO_ALWAYS_RETURN:
types_and_areas += settings.AREAS_TO_ALWAYS_RETURN
types_and_areas_joined = ','.join(
'{area_type_code}-{area_id}'.format(**ta) for ta in types_and_areas
)
area_slugs = [slugify(a[1]['name']) for a in sorted_mapit_results]
ignored_slug = '-'.join(area_slugs)
return {
'type_and_area_ids': types_and_areas_joined,
'ignored_slug': ignored_slug,
}
def _login_url(service):
"""Generates CAS login URL"""
params = {'service': service}
if settings.CAS_RENEW:
params.update({'renew': 'true'})
if settings.CAS_EXTRA_LOGIN_PARAMS:
params.update(settings.CAS_EXTRA_LOGIN_PARAMS)
return urllib_parse.urljoin(settings.CAS_SERVER_URL,
'login') + '?' + urllib_parse.urlencode(params)
def _logout_url(request, next_page=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(settings.CAS_SERVER_URL, 'logout')
if next_page:
protocol = ('http://', 'https://')[request.is_secure()]
host = request.get_host()
url += '?' + urllib_parse.urlencode(
{'url': protocol + host + next_page})
return url
def get_login_url(self):
"""Generates CAS login URL"""
params = {'service': self.service_url}
if self.renew:
params.update({'renew': 'true'})
params.update(self.extra_login_params)
url = urllib_parse.urljoin(self.server_url, 'login')
query = urllib_parse.urlencode(params)
return url + '?' + query
def get_post_elections_from_coords(coords):
url = urljoin(EE_BASE_URL, "/api/elections/?coords={}".format(
urlquote(coords)))
cache_key = 'geolookup-coords:' + coords
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
return get_post_elections(url, cache_key, BadCoordinatesException)
def __make_relative__(self, path, strip=False):
"""
When `relative_paths` is True, make path relative to API Path.
"""
if path:
if SWAGGER_SETTINGS.get('relative_paths', False):
res = UrlParser.__relative_path_matcher__.match(path)
path = urljoin('/', res.groups()[0]) if res else path
return path.strip('/') if strip else path
def get_areas_from_coords(coords):
url = urljoin(EE_BASE_URL,
"/api/elections/?coords={0}".format(urlquote(coords)))
cache_key = 'mapit-postcode:' + coords
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
return get_areas(url, cache_key, BadCoordinatesException)
def __make_relative__(self, path, strip=False):
"""
When `relative_paths` is True, make path relative to API Path.
"""
if path:
if SWAGGER_SETTINGS.get('relative_paths', False):
res = UrlParser.__relative_path_matcher__.match(path)
path = urljoin('/', res.groups()[0]) if res else path
return path.strip('/') if strip else path
def _logout_url(request, next_page=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(settings.CAS_SERVER_URL, "logout")
if next_page:
protocol = get_protocol(request)
host = request.get_host()
next_page_url = urllib_parse.urlunparse((protocol, host, next_page, "", "", ""))
url += "?" + urllib_parse.urlencode({"url": next_page_url})
return url
def get_login_url(self):
"""Generates CAS login URL"""
params = {'service': self.service_url}
if self.renew:
params.update({'renew': 'true'})
params.update(self.extra_login_params)
url = urllib_parse.urljoin(self.server_url, 'login')
query = urllib_parse.urlencode(params)
return url + '?' + query
def check_address(address_string, country=None):
tidied_address_before_country = address_string.strip()
if country is None:
tidied_address = tidied_address_before_country
else:
tidied_address = tidied_address_before_country + ', ' + country
try:
location_results = Geocoder.geocode(tidied_address)
except GeocoderError:
message = _("Failed to find a location for '{0}'")
raise ValidationError(message.format(tidied_address_before_country))
lat, lon = location_results[0].coordinates
all_mapit_json = []
queries_to_try = defaultdict(set)
for election in Election.objects.current().prefetch_related('area_types'):
area_types = election.area_types.values_list('name', flat=True)
queries_to_try[election.area_generation].update(area_types)
for area_generation, area_types in queries_to_try.items():
mapit_lookup_url = urljoin(
settings.MAPIT_BASE_URL, 'point/4326/{lon},{lat}'.format(
lon=lon,
lat=lat,
))
mapit_lookup_url += '?type=' + ','.join(area_types)
mapit_lookup_url += '&generation={}'.format(election.area_generation)
mapit_result = requests.get(mapit_lookup_url,
headers={
'User-Agent': 'scraper/sym',
})
mapit_json = mapit_result.json()
if 'error' in mapit_json:
message = _("The area lookup returned an error: '{error}'")
raise ValidationError(message.format(error=mapit_json['error']))
all_mapit_json += mapit_json.items()
sorted_mapit_results = sorted(all_mapit_json,
key=lambda t: (t[1]['type'], int(t[0])))
if not sorted_mapit_results:
message = _(
"The address '{0}' appears to be outside the area this site knows about"
)
raise ValidationError(message.format(tidied_address_before_country))
types_and_areas = [{
'area_type_code': a[1]['type'],
'area_id': a[0],
} for a in sorted_mapit_results]
if settings.AREAS_TO_ALWAYS_RETURN:
types_and_areas += settings.AREAS_TO_ALWAYS_RETURN
types_and_areas_joined = ','.join(
'{area_type_code}--{area_id}'.format(**ta) for ta in types_and_areas)
area_slugs = [slugify(a[1]['name']) for a in sorted_mapit_results]
ignored_slug = '-'.join(area_slugs)
return {
'type_and_area_ids': types_and_areas_joined,
'ignored_slug': ignored_slug,
}
def get_verification_response(self, ticket):
params = [('ticket', ticket), ('service', self.service_url)]
if self.proxy_callback:
params.append(('pgtUrl', self.proxy_callback))
base_url = urllib_parse.urljoin(self.server_url, self.URL_SUFFIX)
url = base_url + '?' + urllib_parse.urlencode(params)
page = urlopen(url)
try:
return page.read()
finally:
page.close()
def _logout_url(request, next_page=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(settings.CAS_SERVER_URL, 'logout')
if next_page:
protocol = get_protocol(request)
host = request.get_host()
next_page_url = urllib_parse.urlunparse(
(protocol, host, next_page, '', '', ''), )
url += '?' + urllib_parse.urlencode({'url': next_page_url})
return url
def _login_url(service):
"""Generates CAS login URL"""
params = {"service": service}
if settings.CAS_RENEW:
params.update({"renew": "true"})
if settings.CAS_EXTRA_LOGIN_PARAMS:
params.update(settings.CAS_EXTRA_LOGIN_PARAMS)
url = urllib_parse.urljoin(settings.CAS_SERVER_URL, "login")
query = urllib_parse.urlencode(params)
return url + "?" + query
def get(self, request, **kwargs):
latitude = kwargs['latitude']
longitude = kwargs['longitude']
generation_with_types = defaultdict(list)
for t in AreaType.objects.filter(election__current=True) \
.values_list('election__area_generation', 'name'):
generation_with_types[t[0]].append(t[1])
mapit_base_url = urljoin(settings.MAPIT_BASE_URL,
'point/4326/{lon},{lat}'.format(
lon=longitude,
lat=latitude,
))
mapit_json = []
for generation, types in generation_with_types.items():
lookup_url = mapit_base_url + '?type=' \
+ ','.join(sorted(types))
lookup_url += '&generation={0}'.format(generation)
mapit_result = requests.get(lookup_url)
mapit_result = mapit_result.json()
if 'error' in mapit_result:
message = _("The area lookup returned an error: '{0}'") \
.format(mapit_result['error'])
return HttpResponse(
json.dumps({'error': message}),
content_type='application/json',
)
mapit_json += mapit_result.items()
if len(mapit_json) == 0:
message = _("Your location does not seem to be covered by this site")
return HttpResponse(
json.dumps({'error': message}),
content_type='application/json',
)
ids_and_areas = [
"{0}-{1}".format(
area[1]['type'],
mapit.format_code_from_area(area[1]))
for area in mapit_json
]
url = reverse('areas-view', kwargs={
'type_and_area_ids': ','.join(sorted(ids_and_areas))
})
return HttpResponse(
json.dumps({'url': url}),
content_type='application/json',
)
def _logout_url(request, next_page=None):
"""Generates CAS logout URL"""
url = urllib_parse.urljoin(settings.CAS_SERVER_URL, 'logout')
if next_page:
protocol = get_protocol(request)
host = request.get_host()
next_page_url = urllib_parse.urlunparse(
(protocol, host, next_page, '', '', ''),
)
url += '?' + urllib_parse.urlencode({'url': next_page_url})
return url
def _verify_cas2_saml(ticket, service):
"""Verifies CAS 3.0+ XML-based authentication ticket and returns extended attributes.
@date: 2011-11-30
@author: Carlos Gonzalez Vila <*****@*****.**>
Returns username and attributes on success and None,None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
# We do the SAML validation
headers = {'soapaction': 'http://www.oasis-open.org/committees/security',
'cache-control': 'no-cache',
'pragma': 'no-cache',
'accept': 'text/xml',
'connection': 'keep-alive',
'content-type': 'text/xml; charset=utf-8'}
params = [('TARGET', service)]
url = Request(urllib_parse.urljoin(settings.CAS_SERVER_URL, 'samlValidate') + '?' + urllib_parse.urlencode(params), '', headers)
page = urlopen(url, data=get_saml_assertion(ticket))
try:
user = None
attributes = {}
response = page.read()
print(response)
tree = ElementTree.fromstring(response)
# Find the authentication status
success = tree.find('.//' + SAML_1_0_PROTOCOL_NS + 'StatusCode')
if success is not None and success.attrib['Value'] == 'samlp:Success':
# User is validated
attrs = tree.findall('.//' + SAML_1_0_ASSERTION_NS + 'Attribute')
for at in attrs:
if 'uid' in list(at.attrib.values()):
user = at.find(SAML_1_0_ASSERTION_NS + 'AttributeValue').text
attributes['uid'] = user
values = at.findall(SAML_1_0_ASSERTION_NS + 'AttributeValue')
if len(values) > 1:
values_array = []
for v in values:
values_array.append(v.text)
attributes[at.attrib['AttributeName']] = values_array
else:
attributes[at.attrib['AttributeName']] = values[0].text
return user, attributes
finally:
page.close()
def verify_ticket(self, ticket):
"""Verifies CAS 1.0 authentication ticket."""
params = [('ticket', ticket), ('service', self.service)]
url = (urllib_parse.urljoin(self.server_url, 'validate') + '?' +
urllib_parse.urlencode(params))
page = urlopen(url)
try:
verified = page.readline().strip()
if verified == 'yes':
return page.readline().strip(), None, None
else:
return None, None, None
finally:
page.close()
def handle(self, **options):
mapit_url = settings.MAPIT_BASE_URL
self.gb_parties, _ = PartySet.objects.get_or_create(
slug='gb', defaults={'name': 'Great Britain'}
)
self.organizations = {}
self.base_election_info = {
'name': 'Police and Crime Commissioner Elections 2016',
'for_post_role': 'Police and Crime Commissioner',
'label_format': 'Police and Crime Commissioner for {area_name}',
'area_generation': 1,
'election_date': date(2016, 5, 5),
'party_lists_in_use': False,
'mapit_code': 'PDG',
'electon_id_prefix': 'pcc',
# 'area_type_description': 'Police Force',
}
url_path = '/areas/' + self.base_election_info['mapit_code']
url = urljoin(mapit_url, url_path)
r = requests.get(url)
mapit_results = r.json().items()
AREAS_WITHOUT_PCCS = [
"metropolitan",
"city-of-london",
"northern-ireland",
]
# First make all the organisations
for mapit_area_id, mapit_area_data in mapit_results:
if mapit_area_data['codes']['police_id'] in AREAS_WITHOUT_PCCS:
continue
self.add_police_force_orgs(mapit_area_id, mapit_area_data)
# Create a single election
self.create_pcc_election()
# Add all the areas for that election
for mapit_area_id, mapit_area_data in mapit_results:
if mapit_area_data['codes']['police_id'] in AREAS_WITHOUT_PCCS:
# The Met doesn't have a PCCge
continue
self.add_pcc_areas(mapit_area_id, mapit_area_data)
def _verify_cas1(ticket, service):
"""Verifies CAS 1.0 authentication ticket.
Returns username on success and None on failure.
"""
params = [('ticket', ticket), ('service', service)]
url = (urllib_parse.urljoin(settings.CAS_SERVER_URL, 'validate') + '?' +
urllib_parse.urlencode(params))
page = urlopen(url)
try:
verified = page.readline().strip()
if verified == 'yes':
return page.readline().strip(), None
else:
return None, None
finally:
page.close()
def get_areas_from_postcode(original_postcode):
postcode = re.sub(r'(?ms)\s*', '', original_postcode.lower())
if re.search(r'[^a-z0-9]', postcode):
raise BadPostcodeException(
_('There were disallowed characters in "{0}"').format(original_postcode)
)
cache_key = 'mapit-postcode:' + postcode
cached_result = cache.get(cache_key)
if cached_result:
return cached_result
url = urljoin(settings.MAPIT_BASE_URL,
'/postcode/{0}'.format(urlquote(postcode)))
r = requests.get(url)
if r.status_code == 200:
mapit_result = r.json()
result = sorted(
[
(a['type'], format_code_from_area(a))
for a in mapit_result['areas'].values()
if Area.objects.filter(
extra__type__name=a['type'],
identifier=format_code_from_area(a)
).exists()
],
key=area_sort_key
)
cache.set(cache_key, result, settings.MAPIT_CACHE_SECONDS)
return result
elif r.status_code == 400:
mapit_result = r.json()
raise BadPostcodeException(mapit_result['error'])
elif r.status_code == 404:
raise BadPostcodeException(
_('The postcode “{0}” couldn’t be found').format(original_postcode)
)
else:
raise UnknownMapitException(
_('Unknown MapIt error for postcode "{0}"').format(
original_postcode
)
)
def __getitem__(self, key):
key = str(key)
try:
return super(MapitLookup, self).__getitem__(key)
except KeyError:
try:
logging.info("Making extra request to Mapit for ID {0}".format(
key
))
url = urljoin(self.mapit_base_url,
"area/{key}".format(key=key))
req = requests.get(url)
if req.status_code == 404 or \
req.json()[key].get('code', 200) == 404:
raise MapItAreaNotFoundException
self.update({key: req.json()})
return self[key]
except MapItAreaNotFoundException:
raise KeyError
except Exception:
raise
def fetch_saml_validation(self, ticket):
# We do the SAML validation
headers = {
'soapaction': 'http://www.oasis-open.org/committees/security',
'cache-control': 'no-cache',
'pragma': 'no-cache',
'accept': 'text/xml',
'connection': 'keep-alive',
'content-type': 'text/xml; charset=utf-8',
}
params = [('TARGET', self.service_url)]
saml_validate_url = urllib_parse.urljoin(
self.server_url, 'samlValidate',
)
url = Request(
saml_validate_url + '?' + urllib_parse.urlencode(params),
'',
headers,
)
page = urlopen(url, data=self.get_saml_assertion(ticket))
return page
def check_address(address_string, country=None):
tidied_address_before_country = address_string.strip()
if country is None:
tidied_address = tidied_address_before_country
else:
tidied_address = tidied_address_before_country + ", " + country
try:
location_results = Geocoder.geocode(tidied_address)
except GeocoderError:
message = _("Failed to find a location for '{0}'")
raise ValidationError(message.format(tidied_address_before_country))
lat, lon = location_results[0].coordinates
all_mapit_json = []
queries_to_try = defaultdict(set)
for election in Election.objects.current().prefetch_related("area_types"):
area_types = election.area_types.values_list("name", flat=True)
queries_to_try[election.area_generation].update(area_types)
for area_generation, area_types in queries_to_try.items():
mapit_lookup_url = urljoin(settings.MAPIT_BASE_URL, "point/4326/{lon},{lat}".format(lon=lon, lat=lat))
mapit_lookup_url += "?type=" + ",".join(area_types)
mapit_lookup_url += "&generation={0}".format(election.area_generation)
mapit_result = requests.get(mapit_lookup_url, headers={"User-Agent": "scraper/sym"})
mapit_json = mapit_result.json()
if "error" in mapit_json:
message = _("The area lookup returned an error: '{error}'")
raise ValidationError(message.format(error=mapit_json["error"]))
all_mapit_json += mapit_json.items()
sorted_mapit_results = sorted(all_mapit_json, key=lambda t: (t[1]["type"], int(t[0])))
if not sorted_mapit_results:
message = _("The address '{0}' appears to be outside the area this site knows about")
raise ValidationError(message.format(tidied_address_before_country))
types_and_areas = [{"area_type_code": a[1]["type"], "area_id": a[0]} for a in sorted_mapit_results]
if settings.AREAS_TO_ALWAYS_RETURN:
types_and_areas += settings.AREAS_TO_ALWAYS_RETURN
types_and_areas_joined = ",".join("{area_type_code}-{area_id}".format(**ta) for ta in types_and_areas)
area_slugs = [slugify(a[1]["name"]) for a in sorted_mapit_results]
ignored_slug = "-".join(area_slugs)
return {"type_and_area_ids": types_and_areas_joined, "ignored_slug": ignored_slug}
def _verify_cas2(ticket, service):
"""Verifies CAS 2.0+ XML-based authentication ticket.
Returns username on success and None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
params = [('ticket', ticket), ('service', service)]
url = (urllib_parse.urljoin(settings.CAS_SERVER_URL, 'serviceValidate') + '?' +
urllib_parse.urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('authenticationSuccess'):
return tree[0][0].text, None
else:
return None, None
finally:
page.close()
