class StorageWrapper:
def __init__(self):
self.storage = DefaultStorage()
def save(self, filename: str, file_like):
return self.storage.save(filename, file_like)
def list(self, ext: str = None):
_, files = self.storage.listdir("/")
if ext:
files = [f for f in files if f.endswith(ext)]
return files
def exists(self, filename: str):
return self.storage.exists(filename)
def find(self, filename: str) -> Optional[str]:
for file_ in self.list():
if file_ == filename:
return file_
return None
def write_to(self, filename: str, callback: Any):
f = self.storage.open(filename)
callback(f.read())
f.close()
def remove(self, filename: str):
self.storage.delete(filename)
def render(self, context):
challenge_short_name = context.page.challenge.short_name
projectpath = challenge_short_name + "/" + self.path
storage = DefaultStorage()
try:
filenames = storage.listdir(projectpath)[1]
except OSError as e:
return self.make_dataset_error_msg(str(e))
filenames.sort()
# if extensionsFilter is given, show only filenames with those extensions
if "extensionFilter" in self.args.keys():
extensions = self.args["extensionFilter"].split(",")
filenames = filter_by_extension(filenames, extensions)
links = []
for filename in filenames:
downloadlink = reverse(
"serving:challenge-file",
kwargs={
"challenge_short_name": challenge_short_name,
"path": f"{self.path}/{filename}",
},
)
links.append('<li><a href="' + downloadlink + '">' + filename +
" </a></li>")
htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>"
return htmlOut
def render(self, context):
challenge: Challenge = context["currentpage"].challenge
try:
projectpath = safe_join(challenge.get_project_data_folder(),
self.path)
except SuspiciousFileOperation:
return self.make_dataset_error_msg(
"path is outside the challenge folder.")
storage = DefaultStorage()
try:
filenames = storage.listdir(projectpath)[1]
except OSError as e:
return self.make_dataset_error_msg(str(e))
filenames.sort()
# if extensionsFilter is given, show only filenames with those extensions
if "extensionFilter" in self.args.keys():
extensions = self.args["extensionFilter"].split(",")
filenames = filter_by_extension(filenames, extensions)
links = []
for filename in filenames:
downloadlink = reverse(
"root-serving:challenge-file",
kwargs={
"challenge_name": challenge.short_name,
"path": f"{self.path}/{filename}",
},
)
links.append('<li><a href="' + downloadlink + '">' + filename +
" </a></li>")
htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>"
return htmlOut
def get_dirnames(path):
""" Get all directory names in path as list of strings
Raises: OSError if directory can not be found
"""
storage = DefaultStorage()
dirnames = storage.listdir(path)[0]
dirnames.sort()
return dirnames
def get_dirnames(path):
""" Get all directory names in path as list of strings
Raises: OSError if directory can not be found
"""
storage = DefaultStorage()
dirnames = storage.listdir(path)[0]
dirnames.sort()
return dirnames
def cleanup_files(self):
"""Remove stale screenshots"""
storage = DefaultStorage()
try:
files = storage.listdir('screenshots')[1]
except OSError:
return
for name in files:
fullname = os.path.join('screenshots', name)
if not Screenshot.objects.filter(image=fullname).exists():
storage.delete(fullname)
def cleanup_screenshot_files():
"""Remove stale screenshots"""
storage = DefaultStorage()
try:
files = storage.listdir('screenshots')[1]
except OSError:
return
for name in files:
fullname = os.path.join('screenshots', name)
if not Screenshot.objects.filter(image=fullname).exists():
storage.delete(fullname)
def get_filenames(self, context, path):
""" Get all filenames in path
Raises OSError if directory can not be found
"""
challenge_short_name = context.page.challenge.short_name
projectpath = challenge_short_name + "/" + path
storage = DefaultStorage()
filenames = storage.listdir(projectpath)[1]
filenames.sort()
# if extensionsFilter is given, show only filenames with those extensions
if "extensionFilter" in self.args.keys():
extensions = self.args["extensionFilter"].split(",")
filenames = filter_by_extension(filenames, extensions)
return filenames
def render(self, context):
challenge: Challenge = context["currentpage"].challenge
try:
projectpath = safe_join(
challenge.get_project_data_folder(), self.path
)
except SuspiciousFileOperation:
return self.make_dataset_error_msg(
"path is outside the challenge folder."
)
storage = DefaultStorage()
try:
filenames = storage.listdir(projectpath)[1]
except OSError as e:
return self.make_dataset_error_msg(str(e))
filenames.sort()
# if extensionsFilter is given, show only filenames with those extensions
if "extensionFilter" in self.args.keys():
extensions = self.args["extensionFilter"].split(",")
filenames = filter_by_extension(filenames, extensions)
links = []
for filename in filenames:
downloadlink = reverse(
"root-serving:challenge-file",
kwargs={
"challenge_name": challenge.short_name,
"path": f"{self.path}/{filename}",
},
)
links.append(
'<li><a href="' + downloadlink + '">' + filename + " </a></li>"
)
htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>"
return htmlOut
class TestAPI(TmpMediaRootMixin, UK2015ExamplesMixin, WebTest):
def setUp(self):
super().setUp()
person_extra = PersonExtraFactory.create(base__id="2009",
base__name="Tessa Jowell")
dulwich_not_stand = PersonExtraFactory.create(base__id="4322",
base__name="Helen Hayes")
edinburgh_candidate = PersonExtraFactory.create(
base__id="818", base__name="Sheila Gilmore")
edinburgh_winner = PersonExtraFactory.create(
base__id="5795", base__name="Tommy Sheppard")
edinburgh_may_stand = PersonExtraFactory.create(
base__id="5163", base__name="Peter McColl")
MembershipFactory.create(
person=person_extra.base,
post=self.dulwich_post_extra.base,
on_behalf_of=self.labour_party_extra.base,
post_election=self.dulwich_post_extra_pee,
)
MembershipFactory.create(
person=person_extra.base,
organization=self.labour_party_extra.base,
post_election=self.edinburgh_east_post_extra_pee,
)
MembershipFactory.create(
person=dulwich_not_stand.base,
post=self.dulwich_post_extra.base,
on_behalf_of=self.labour_party_extra.base,
post_election=self.dulwich_post_extra_pee_earlier,
)
dulwich_not_stand.not_standing.add(self.election)
MembershipFactory.create(
person=edinburgh_winner.base,
post=self.edinburgh_east_post_extra.base,
on_behalf_of=self.labour_party_extra.base,
elected=True,
post_election=self.edinburgh_east_post_extra_pee,
)
MembershipFactory.create(
person=edinburgh_candidate.base,
post=self.edinburgh_east_post_extra.base,
on_behalf_of=self.labour_party_extra.base,
post_election=self.edinburgh_east_post_extra_pee,
)
MembershipFactory.create(
person=edinburgh_may_stand.base,
post=self.edinburgh_east_post_extra.base,
on_behalf_of=self.labour_party_extra.base,
post_election=self.edinburgh_east_post_extra_pee_earlier,
)
self.storage = DefaultStorage()
def test_api_basic_response(self):
response = self.app.get("/api/v0.9/")
self.assertEqual(response.status_code, 200)
json = response.json
self.assertEqual(json["persons"],
"http://localhost:80/api/v0.9/persons/")
self.assertEqual(json["organizations"],
"http://localhost:80/api/v0.9/organizations/")
self.assertEqual(json["elections"],
"http://localhost:80/api/v0.9/elections/")
self.assertEqual(json["posts"], "http://localhost:80/api/v0.9/posts/")
persons_resp = self.app.get("/api/v0.9/persons/")
self.assertEqual(persons_resp.status_code, 200)
organizations_resp = self.app.get("/api/v0.9/organizations/")
self.assertEqual(organizations_resp.status_code, 200)
elections_resp = self.app.get("/api/v0.9/elections/")
self.assertEqual(elections_resp.status_code, 200)
posts_resp = self.app.get("/api/v0.9/posts/")
self.assertEqual(posts_resp.status_code, 200)
def test_api_errors(self):
response = self.app.get("/api/", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.get("/api/v0.8", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.get("/api/v0.9/person/", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.get("/api/v0.9/persons/4000/", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.post("/api/v0.9/persons/", {}, expect_errors=True)
self.assertEqual(response.status_code, 403)
def test_api_persons(self):
persons_resp = self.app.get("/api/v0.9/persons/")
persons = persons_resp.json
self.assertEqual(persons["count"], len(persons["results"]))
self.assertEqual(persons["count"], 5)
def test_api_person(self):
person_resp = self.app.get("/api/v0.9/persons/2009/")
self.assertEqual(person_resp.status_code, 200)
person = person_resp.json
self.assertEqual(person["id"], 2009)
self.assertEqual(person["name"], "Tessa Jowell")
memberships = sorted(person["memberships"], key=lambda m: m["role"])
self.assertEqual(len(memberships), 2)
self.assertEqual(memberships[1]["role"], "Candidate")
self.assertEqual(len(person["versions"]), 0)
def test_api_organizations(self):
organizations_resp = self.app.get("/api/v0.9/organizations/")
organizations = organizations_resp.json
self.assertEqual(organizations["count"], len(organizations["results"]))
self.assertEqual(organizations["count"], 7)
def test_api_organization(self):
organizations_resp = self.app.get("/api/v0.9/organizations/")
organizations = organizations_resp.json
organization_url = None
for organization in organizations["results"]:
if organization["id"] == "party:53":
organization_url = organization["url"]
break
organization_resp = self.app.get(organization_url)
self.assertEqual(organization_resp.status_code, 200)
organization = organization_resp.json
self.assertEqual(organization["id"], "party:53")
self.assertEqual(organization["name"], "Labour Party")
def test_api_elections(self):
elections_resp = self.app.get("/api/v0.9/elections/")
elections = elections_resp.json
self.assertEqual(elections["count"], len(elections["results"]))
self.assertEqual(elections["count"], 3)
def test_api_election(self):
elections_resp = self.app.get("/api/v0.9/elections/")
elections = elections_resp.json
election_url = None
for election in elections["results"]:
if election["id"] == "2015":
election_url = election["url"]
break
election_resp = self.app.get(election_url)
self.assertEqual(election_resp.status_code, 200)
election = election_resp.json
self.assertEqual(election["id"], "2015")
self.assertEqual(election["name"], "2015 General Election")
def test_api_posts(self):
posts_resp = self.app.get("/api/v0.9/posts/")
posts = posts_resp.json
self.assertEqual(posts["count"], len(posts["results"]))
self.assertEqual(posts["count"], 5)
def test_api_post(self):
posts_resp = self.app.get("/api/v0.9/posts/")
posts = posts_resp.json
post_url = None
for post in posts["results"]:
if post["id"] == "65808":
post_url = post["url"]
break
self.assertTrue(post_url)
post_resp = self.app.get(post_url)
self.assertEqual(post_resp.status_code, 200)
post = post_resp.json
self.assertEqual(post["id"], "65808")
self.assertEqual(post["label"],
"Member of Parliament for Dulwich and West Norwood")
def test_api_person_redirects(self):
PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
PersonRedirect.objects.create(old_person_id="5678", new_person_id="12")
person_redirects_resp = self.app.get("/api/v0.9/person_redirects/")
person_redirects = person_redirects_resp.json
self.assertEqual(person_redirects["results"][0]["old_person_id"], 1234)
self.assertEqual(person_redirects["results"][0]["new_person_id"], 42)
self.assertEqual(person_redirects["results"][1]["old_person_id"], 5678)
self.assertEqual(person_redirects["results"][1]["new_person_id"], 12)
def test_api_person_redirect(self):
PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
url = "/api/v0.9/person_redirects/1234/"
person_redirect_resp = self.app.get(url)
person_redirect = person_redirect_resp.json
self.assertEqual(person_redirect["old_person_id"], 1234)
self.assertEqual(person_redirect["new_person_id"], 42)
def test_api_version_info(self):
version_resp = self.app.get("/version.json")
self.assertEqual(version_resp.status_code, 200)
info = version_resp.json
self.assertEqual(info["users_who_have_edited"], 0)
self.assertEqual(info["interesting_user_actions"], 0)
LoggedAction.objects.create(action_type="set-candidate-not-elected")
LoggedAction.objects.create(action_type="edit-candidate")
version_resp = self.app.get("/version.json")
info = version_resp.json
self.assertEqual(info["interesting_user_actions"], 1)
def test_api_cors_headers(self):
resp = self.app.get("/api/v0.9/",
headers={"Origin": b"http://example.com"})
self.assertTrue("Access-Control-Allow-Origin" in resp.headers)
self.assertEqual(resp.headers["Access-Control-Allow-Origin"], "*")
resp = self.app.get("/", headers={"Origin": b"http://example.com"})
self.assertFalse("Access-Control-Allow-Origin" in resp.headers)
def test_api_jsonp_response(self):
response = self.app.get("/api/v0.9/?format=jsonp&callback=test")
self.assertEqual(response.status_code, 200)
self.assertTrue(response.text.startswith("test("))
@patch(
"candidates.management.commands.candidates_cache_api_to_directory.datetime"
)
def test_persons_api_to_directory(self, mock_datetime):
# current
# timestamped
# timestamped
mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
target_directory = settings.MEDIA_ROOT
call_command(
"candidates_cache_api_to_directory",
page_size="3",
url_prefix="https://example.com/media/api-cache-for-wcivf/",
)
expected_leafname = "2017-05-14T12:33:05"
expected_timestamped_directory = join(settings.MEDIA_ROOT,
"cached-api", expected_leafname)
expected_path = join("cached-api", "latest")
self.assertTrue(self.storage.exists(expected_timestamped_directory))
self.assertTrue(self.storage.exists(expected_path))
# Check that the files in that directory are as expected:
entries = self.storage.listdir(expected_timestamped_directory)[1]
persons_1_leafname = "persons-000001.json"
persons_2_leafname = "persons-000002.json"
posts_1_leafname = "posts-000001.json"
posts_2_leafname = "posts-000002.json"
self.assertEqual(
set(entries),
{
persons_1_leafname,
persons_2_leafname,
posts_1_leafname,
posts_2_leafname,
},
)
# Get the data from those pages:
with self.storage.open(
join(expected_timestamped_directory, persons_1_leafname)) as f:
persons_1_data = json.loads(f.read().decode('utf8'))
with self.storage.open(
join(expected_timestamped_directory, persons_2_leafname)) as f:
persons_2_data = json.loads(f.read().decode('utf8'))
with self.storage.open(
join(expected_timestamped_directory, posts_1_leafname)) as f:
posts_1_data = json.loads(f.read().decode('utf8'))
with self.storage.open(
join(expected_timestamped_directory, posts_2_leafname)) as f:
posts_2_data = json.loads(f.read().decode('utf8'))
# Check the previous and next links are as we expect:
self.assertEqual(
persons_1_data["next"],
"https://example.com/media/api-cache-for-wcivf/{}/{}".format(
expected_leafname, persons_2_leafname),
)
self.assertEqual(persons_1_data["previous"], None)
self.assertEqual(persons_2_data["next"], None)
self.assertEqual(
persons_2_data["previous"],
"https://example.com/media/api-cache-for-wcivf/{}/{}".format(
expected_leafname, persons_1_leafname),
)
self.assertEqual(
posts_1_data["next"],
"https://example.com/media/api-cache-for-wcivf/{}/{}".format(
expected_leafname, posts_2_leafname),
)
self.assertEqual(posts_1_data["previous"], None)
self.assertEqual(posts_2_data["next"], None)
self.assertEqual(
posts_2_data["previous"],
"https://example.com/media/api-cache-for-wcivf/{}/{}".format(
expected_leafname, posts_1_leafname),
)
# Check that the URL of the first person is as expected,
# as well as it being the right person:
first_person = persons_1_data["results"][0]
self.assertEqual(first_person["id"], 818)
self.assertEqual(first_person["name"], "Sheila Gilmore")
self.assertEqual(
first_person["url"],
"https://candidates.democracyclub.org.uk/api/v0.9/persons/818/?format=json",
)
# Similarly, check that the URL of the first post is as expected:
first_post = posts_1_data["results"][0]
self.assertEqual(first_post["id"], self.edinburgh_east_post_extra.slug)
self.assertEqual(first_post["label"],
"Member of Parliament for Edinburgh East")
self.assertEqual(
first_post["url"],
"https://candidates.democracyclub.org.uk/api/v0.9/posts/14419/?format=json",
)
def _setup_cached_api_directory(self, dir_list):
"""
Saves a tmp file in settings.MEDIA_ROOT, called `.keep` in each
directory in dir_list.
"""
for d in dir_list:
self.storage.save(join("cached-api", d, ".keep"), ContentFile("."))
@patch(
"candidates.management.commands.candidates_cache_api_to_directory.datetime"
)
def test_cache_api_to_directory_prune(self, mock_datetime):
# Need to make sure datetime.strptime still works:
mock_datetime.strptime.side_effect = datetime.strptime
mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
expected_to_prune = [
"2017-05-12T08:00:00",
"2017-05-12T10:00:00",
"2017-05-12T12:00:00",
]
expected_to_keep = [
"2017-05-14T08:00:00",
"2017-05-14T10:00:00",
"2017-05-14T12:00:00",
"2017-05-14T12:33:05",
]
self._setup_cached_api_directory(expected_to_keep + expected_to_prune)
self.assertTrue(
self.storage.exists(
join("cached-api", "2017-05-12T08:00:00", ".keep")))
call_command(
"candidates_cache_api_to_directory",
page_size="3",
url_prefix="https://example.com/media/api-cache-for-wcivf/",
prune=True,
)
for dir_name in expected_to_keep:
self.assertTrue(
self.storage.exists(join("cached-api", dir_name, ".keep")))
for dir_name in expected_to_prune:
self.assertFalse(
self.storage.exists(join("cached-api", dir_name, ".keep")))
@patch(
"candidates.management.commands.candidates_cache_api_to_directory.datetime"
)
def test_cache_api_to_directory_prune_four_old(self, mock_datetime):
# Need to make sure datetime.strptime still works:
mock_datetime.strptime.side_effect = datetime.strptime
mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
expected_to_prune = ["2017-05-12T06:00:00"]
expected_to_keep = [
"2017-05-12T08:00:00",
"2017-05-12T10:00:00",
"2017-05-12T12:00:00",
"2017-05-14T12:33:05",
]
self._setup_cached_api_directory(expected_to_keep + expected_to_prune)
self.assertTrue(
self.storage.exists(
join("cached-api", "2017-05-12T06:00:00", ".keep")))
self.assertTrue(
self.storage.exists(
join("cached-api", "2017-05-12T08:00:00", ".keep")))
call_command(
"candidates_cache_api_to_directory",
page_size="3",
url_prefix="https://example.com/media/api-cache-for-wcivf/",
prune=True,
)
# Even though all of those directories are more than 36
# hours old, they should all be kept because they're the
# most recent 4:
for dir_name in expected_to_keep:
self.assertTrue(
self.storage.exists(join("cached-api", dir_name, ".keep")))
for dir_name in expected_to_prune:
self.assertFalse(
self.storage.exists(join("cached-api", dir_name, ".keep")))
class CSVTests(TmpMediaRootMixin, TestUserMixin, UK2015ExamplesMixin, TestCase):
def setUp(self):
super().setUp()
self.storage = DefaultStorage()
# The second person's name (and party name) have diacritics in
# them to test handling of Unicode when outputting to CSV.
self.gb_person = people.tests.factories.PersonFactory.create(
id=2009,
name="Tessa Jowell",
honorific_suffix="DBE",
honorific_prefix="Ms",
gender="female",
)
PersonImage.objects.create_from_file(
EXAMPLE_IMAGE_FILENAME,
"images/jowell-pilot.jpg",
defaults={
"person": self.gb_person,
"is_primary": True,
"source": "Taken from Wikipedia",
"copyright": "example-license",
"uploading_user": self.user,
"user_notes": "A photo of Tessa Jowell",
},
)
self.ni_person = people.tests.factories.PersonFactory.create(
id=1953, name="Daithà McKay", gender="male"
)
north_antrim_post = factories.PostFactory.create(
elections=(self.election, self.earlier_election),
organization=self.commons,
slug="66135",
label="Member of Parliament for North Antrim",
party_set=self.ni_parties,
)
factories.MembershipFactory.create(
person=self.ni_person,
post=north_antrim_post,
party=self.sinn_fein,
post_election=self.election.postextraelection_set.get(
post=north_antrim_post
),
)
factories.MembershipFactory.create(
person=self.ni_person,
post=north_antrim_post,
party=self.sinn_fein,
post_election=self.earlier_election.postextraelection_set.get(
post=north_antrim_post
),
)
factories.MembershipFactory.create(
person=self.gb_person,
post=self.camberwell_post,
party=self.labour_party,
post_election=self.camberwell_post_pee,
)
factories.MembershipFactory.create(
person=self.gb_person,
post=self.dulwich_post,
party=self.labour_party,
post_election=self.dulwich_post_pee_earlier,
)
self.gb_person.tmp_person_identifiers.create(
internal_identifier="10326", value_type="theyworkforyou"
)
self.gb_person.tmp_person_identifiers.create(
value_type="email", value="*****@*****.**"
)
def test_as_single_dict(self):
membership = (
Membership.objects.for_csv()
.filter(person=self.gb_person.id)
.first()
)
# After the select_related and prefetch_related calls
# PersonExtra there should only be one more query - that to
# find the complex fields mapping:
with self.assertNumQueries(0):
membership_dict = membership.dict_for_csv()
self.assertEqual(
sorted(list(membership_dict.keys())),
sorted(settings.CSV_ROW_FIELDS),
)
self.assertEqual(len(membership_dict.keys()), 39)
self.assertEqual(membership_dict["id"], 2009)
self.assertEqual(
membership_dict["parlparse_id"], "uk.org.publicwhip/person/10326"
)
self.assertEqual(
membership_dict["theyworkforyou_url"],
"http://www.theyworkforyou.com/mp/10326",
)
def test_as_dict_2010(self):
with self.assertNumQueries(5):
memberships_dicts, elected = memberships_dicts_for_csv(
self.earlier_election.slug
)
self.assertEqual(len(memberships_dicts[self.earlier_election.slug]), 2)
membership_dict = memberships_dicts["parl.2010-05-06"][1]
self.assertEqual(len(membership_dict.keys()), 39)
self.assertEqual(membership_dict["id"], 2009)
def test_csv_output(self):
tessa_image_url = self.gb_person.primary_image.url
d = {
"election_date": date_in_near_future,
"earlier_election_date": date_in_near_future
- timedelta(days=FOUR_YEARS_IN_DAYS),
}
PersonRedirect.objects.create(old_person_id=12, new_person_id=1953)
PersonRedirect.objects.create(old_person_id=56, new_person_id=1953)
self.maxDiff = None
example_output = (
"id,name,honorific_prefix,honorific_suffix,gender,birth_date,election,party_id,party_name,post_id,post_label,mapit_url,elected,email,twitter_username,facebook_page_url,party_ppc_page_url,facebook_personal_url,homepage_url,wikipedia_url,linkedin_url,image_url,proxy_image_url_template,image_copyright,image_uploading_user,image_uploading_user_notes,twitter_user_id,election_date,election_current,party_lists_in_use,party_list_position,old_person_ids,gss_code,parlparse_id,theyworkforyou_url,party_ec_id,favourite_biscuits,cancelled_poll,wikidata_url\r\n"
+ "1953,Daith\xed McKay,,,male,,parl.2010-05-06,party:39,Sinn F\xe9in,66135,North Antrim,,,,,,,,,,,,,,,,,{earlier_election_date},False,False,,12;56,,,,PP39,,False,\r\n".format(
**d
)
+ "2009,Tessa Jowell,Ms,DBE,female,,parl.2010-05-06,party:53,Labour Party,65808,Dulwich and West Norwood,,,[email protected],,,,,,,,{image_url},,example-license,john,A photo of Tessa Jowell,,{earlier_election_date},False,False,,,,uk.org.publicwhip/person/10326,http://www.theyworkforyou.com/mp/10326,PP53,,False,\r\n".format(
image_url=tessa_image_url, **d
)
+ "1953,Daith\xed McKay,,,male,,parl.2015-05-07,party:39,Sinn F\xe9in,66135,North Antrim,,,,,,,,,,,,,,,,,{election_date},True,False,,12;56,,,,PP39,,False,\r\n".format(
**d
)
+ "2009,Tessa Jowell,Ms,DBE,female,,parl.2015-05-07,party:53,Labour Party,65913,Camberwell and Peckham,,,[email protected],,,,,,,,{image_url},,example-license,john,A photo of Tessa Jowell,,{election_date},True,False,,,,uk.org.publicwhip/person/10326,http://www.theyworkforyou.com/mp/10326,PP53,,False,\r\n".format(
image_url=tessa_image_url, **d
)
)
with self.assertNumQueries(5):
memberships_dicts, elected = memberships_dicts_for_csv()
all_members = []
for slug, members in memberships_dicts.items():
all_members += members
self.assertEqual(list_to_csv(all_members), example_output)
def test_create_csv_management_command(self):
# An empty media directory
self.assertEqual(
self.storage.listdir(self.storage.base_location)[1], []
)
# We expect a CSV file per election, and one for all elections
call_command("candidates_create_csv")
self.assertEqual(
set(sorted(self.storage.listdir(".")[1])),
set(
[
"candidates-parl.2010-05-06.csv",
"candidates-parl.2015-05-07.csv",
"candidates-all.csv",
"candidates-elected-all.csv",
"candidates-local.maidstone.2016-05-05.csv",
"candidates-2015-05-07.csv",
"candidates-2010-05-06.csv",
]
),
)
def test_create_csv_management_command_single_election(self):
# An empty media directory
self.assertEqual(
self.storage.listdir(self.storage.base_location), (["images"], [])
)
# We expect a single CSV file
call_command("candidates_create_csv", "--election", "parl.2015-05-07")
self.assertSetEqual(
set(self.storage.listdir(".")[1]),
set(
[
"candidates-parl.2010-05-06.csv",
"candidates-parl.2015-05-07.csv",
"candidates-local.maidstone.2016-05-05.csv",
"candidates-2015-05-07.csv",
]
),
)
def test_create_csv_management_command_single_election_doesnt_exist(self):
# An empty media directory, apart from the images dir
self.assertEqual(
self.storage.listdir(self.storage.base_location), (["images"], [])
)
with self.assertRaises(CommandError):
call_command("candidates_create_csv", "--election", "foo")
def test_create_csv_no_candidates_for_election(self):
# An empty media directory, apart from the images dir
self.assertEqual(
self.storage.listdir(self.storage.base_location), (["images"], [])
)
factories.ElectionFactory.create(
slug="2018",
name="2018 General Election",
for_post_role="Member of Parliament",
organization=self.commons,
)
call_command("candidates_create_csv")
self.assertSetEqual(
set(self.storage.listdir(".")[1]),
set(
[
"candidates-parl.2010-05-06.csv",
"candidates-parl.2015-05-07.csv",
"candidates-2018.csv",
"candidates-all.csv",
"candidates-elected-all.csv",
"candidates-local.maidstone.2016-05-05.csv",
"candidates-2015-05-07.csv",
"candidates-2010-05-06.csv",
]
),
)
empty_file = self.storage.open("candidates-2018.csv").read()
self.assertEqual(len(empty_file.splitlines()), 1)
self.assertEqual(
empty_file.splitlines()[0].decode(),
(",".join(settings.CSV_ROW_FIELDS)),
)
non_empty_file = self.storage.open(
"candidates-parl.2015-05-07.csv"
).read()
self.assertEqual(len(non_empty_file.splitlines()), 3)
class TestAPI(TestUserMixin, TmpMediaRootMixin, UK2015ExamplesMixin, WebTest):
def setUp(self):
super().setUp()
person = PersonFactory.create(id=2009, name="Tessa Jowell")
PersonImage.objects.update_or_create_from_file(
EXAMPLE_IMAGE_FILENAME,
"images/imported.jpg",
person,
defaults={
"md5sum": "md5sum",
"copyright": "example-license",
"uploading_user": self.user,
"user_notes": "Here's an image...",
"is_primary": True,
"source": "Found on the candidate's Flickr feed",
},
)
dulwich_not_stand = PersonFactory.create(id=4322, name="Helen Hayes")
edinburgh_candidate = PersonFactory.create(id="818",
name="Sheila Gilmore")
edinburgh_winner = PersonFactory.create(id="5795",
name="Tommy Sheppard")
edinburgh_may_stand = PersonFactory.create(id="5163",
name="Peter McColl")
MembershipFactory.create(
person=person,
post=self.dulwich_post,
party=self.labour_party,
ballot=self.dulwich_post_ballot,
)
MembershipFactory.create(person=person,
ballot=self.edinburgh_east_post_ballot)
MembershipFactory.create(
person=dulwich_not_stand,
post=self.dulwich_post,
party=self.labour_party,
ballot=self.dulwich_post_ballot_earlier,
)
dulwich_not_stand.not_standing.add(self.election)
MembershipFactory.create(
person=edinburgh_winner,
post=self.edinburgh_east_post,
party=self.labour_party,
elected=True,
ballot=self.edinburgh_east_post_ballot,
)
MembershipFactory.create(
person=edinburgh_candidate,
post=self.edinburgh_east_post,
party=self.labour_party,
ballot=self.edinburgh_east_post_ballot,
)
MembershipFactory.create(
person=edinburgh_may_stand,
post=self.edinburgh_east_post,
party=self.labour_party,
ballot=self.edinburgh_east_post_ballot_earlier,
)
self.storage = DefaultStorage()
def test_api_basic_response(self):
response = self.app.get("/api/v0.9/")
self.assertEqual(response.status_code, 200)
json = response.json
self.assertEqual(json["persons"],
"http://testserver/api/v0.9/persons/")
self.assertEqual(json["organizations"],
"http://testserver/api/v0.9/organizations/")
self.assertEqual(json["elections"],
"http://testserver/api/v0.9/elections/")
self.assertEqual(json["posts"], "http://testserver/api/v0.9/posts/")
persons_resp = self.app.get("/api/v0.9/persons/")
self.assertEqual(persons_resp.status_code, 200)
organizations_resp = self.app.get("/api/v0.9/organizations/")
self.assertEqual(organizations_resp.status_code, 200)
elections_resp = self.app.get("/api/v0.9/elections/")
self.assertEqual(elections_resp.status_code, 200)
posts_resp = self.app.get("/api/v0.9/posts/")
self.assertEqual(posts_resp.status_code, 200)
def test_api_home(self):
response = self.app.get("/api/")
self.assertEqual(response.status_code, 302)
response = self.app.get("/api/docs/")
self.assertEqual(response.status_code, 200)
response = self.app.get("/api/v0.8", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.get("/api/v0.9/person/", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.get("/api/v0.9/persons/4000/", expect_errors=True)
self.assertEqual(response.status_code, 404)
response = self.app.post("/api/v0.9/persons/",
params={},
expect_errors=True)
self.assertEqual(response.status_code, 403)
def test_api_persons(self):
persons_resp = self.app.get("/api/v0.9/persons/")
persons = persons_resp.json
self.assertEqual(persons["count"], len(persons["results"]))
self.assertEqual(persons["count"], 5)
def test_api_person(self):
person_resp = self.app.get("/api/v0.9/persons/2009/")
self.assertEqual(person_resp.status_code, 200)
person = person_resp.json
self.assertEqual(person["id"], 2009)
self.assertEqual(person["name"], "Tessa Jowell")
memberships = sorted(person["memberships"], key=lambda m: m["role"])
self.assertEqual(len(memberships), 2)
self.assertEqual(memberships[1]["role"], "Candidate")
self.assertEqual(len(person["versions"]), 0)
def _make_legacy_parties(self):
"""
It used to be that political parties were stored on the "Organization"
model. for maintaining v0.9 API compatibility we've not deleted them
from that model (yet), so let's make the test data support this legacy
case
"""
from candidates.tests.factories import OrganizationFactory
from candidates.tests.uk_examples import EXAMPLE_PARTIES
for party in EXAMPLE_PARTIES:
p = OrganizationFactory(slug=party["legacy_slug"],
name=party["name"])
def test_api_legacy_organizations_with_parties(self):
self._make_legacy_parties()
organizations_resp = self.app.get("/api/v0.9/organizations/")
organizations = organizations_resp.json
self.assertEqual(organizations["count"], len(organizations["results"]))
self.assertEqual(organizations["count"], 7)
def test_api_legacy_organization_with_parties(self):
self._make_legacy_parties()
organizations_resp = self.app.get("/api/v0.9/organizations/")
organizations = organizations_resp.json
organization_url = None
for organization in organizations["results"]:
if organization["id"] == "party:53":
organization_url = organization["url"]
break
organization_resp = self.app.get(organization_url)
self.assertEqual(organization_resp.status_code, 200)
organization = organization_resp.json
self.assertEqual(organization["id"], "party:53")
self.assertEqual(organization["name"], "Labour Party")
def test_api_elections(self):
elections_resp = self.app.get("/api/v0.9/elections/")
elections = elections_resp.json
self.assertEqual(elections["count"], len(elections["results"]))
self.assertEqual(elections["count"], 3)
def test_api_elections_without_orgs(self):
# Regression test that we can serve elections without an organzation
self.election.organization = None
self.election.save()
elections_resp = self.app.get("/api/v0.9/elections/",
expect_errors=True)
self.assertEqual(elections_resp.status_code, 200)
def test_api_election(self):
elections_resp = self.app.get("/api/v0.9/elections/")
elections = elections_resp.json
election_url = None
for election in elections["results"]:
if election["id"] == "parl.2015-05-07":
election_url = election["url"]
break
election_resp = self.app.get(election_url)
self.assertEqual(election_resp.status_code, 200)
election = election_resp.json
self.assertEqual(election["id"], "parl.2015-05-07")
self.assertEqual(election["name"], "2015 General Election")
def test_api_posts(self):
posts_resp = self.app.get("/api/v0.9/posts/")
posts = posts_resp.json
self.assertEqual(posts["count"], len(posts["results"]))
self.assertEqual(posts["count"], 5)
def test_api_post(self):
posts_resp = self.app.get("/api/v0.9/posts/")
posts = posts_resp.json
post_url = None
for post in posts["results"]:
if post["id"] == "65808":
post_url = post["url"]
break
self.assertTrue(post_url)
post_resp = self.app.get(post_url)
self.assertEqual(post_resp.status_code, 200)
post = post_resp.json
self.assertEqual(post["id"], "65808")
self.assertEqual(post["label"],
"Member of Parliament for Dulwich and West Norwood")
def test_api_person_redirects(self):
PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
PersonRedirect.objects.create(old_person_id="5678", new_person_id="12")
person_redirects_resp = self.app.get("/api/v0.9/person_redirects/")
person_redirects = person_redirects_resp.json
self.assertEqual(person_redirects["results"][0]["old_person_id"], 1234)
self.assertEqual(person_redirects["results"][0]["new_person_id"], 42)
self.assertEqual(person_redirects["results"][1]["old_person_id"], 5678)
self.assertEqual(person_redirects["results"][1]["new_person_id"], 12)
def test_api_person_redirect(self):
PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
url = "/api/v0.9/person_redirects/1234/"
person_redirect_resp = self.app.get(url)
person_redirect = person_redirect_resp.json
self.assertEqual(person_redirect["old_person_id"], 1234)
self.assertEqual(person_redirect["new_person_id"], 42)
def test_api_version_info(self):
version_resp = self.app.get("/version.json")
self.assertEqual(version_resp.status_code, 200)
info = version_resp.json
self.assertEqual(info["users_who_have_edited"], 0)
self.assertEqual(info["interesting_user_actions"], 0)
LoggedAction.objects.create(action_type="set-candidate-not-elected")
LoggedAction.objects.create(action_type="edit-candidate")
version_resp = self.app.get("/version.json")
info = version_resp.json
self.assertEqual(info["interesting_user_actions"], 1)
def test_api_cors_headers(self):
resp = self.app.get("/api/v0.9/",
headers={"Origin": b"http://example.com"})
self.assertTrue("Access-Control-Allow-Origin" in resp.headers)
self.assertEqual(resp.headers["Access-Control-Allow-Origin"], "*")
resp = self.app.get("/", headers={"Origin": b"http://example.com"})
self.assertFalse("Access-Control-Allow-Origin" in resp.headers)
def test_api_jsonp_response(self):
response = self.app.get("/api/v0.9/?format=jsonp&callback=test")
self.assertEqual(response.status_code, 200)
self.assertTrue(response.text.startswith("test("))
@patch(
"candidates.management.commands.candidates_cache_api_to_directory.datetime"
)
def test_persons_api_to_directory(self, mock_datetime):
# current
# timestamped
# timestamped
mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
target_directory = settings.MEDIA_ROOT
call_command(
"candidates_cache_api_to_directory",
page_size="3",
url_prefix="https://example.com/media/api-cache-for-wcivf",
)
expected_leafname = "2017-05-14T12:33:05"
expected_timestamped_directory = join(settings.MEDIA_ROOT,
"cached-api", expected_leafname)
expected_path = join("cached-api", "latest")
self.assertTrue(self.storage.exists(expected_timestamped_directory))
self.assertTrue(self.storage.exists(expected_path))
# Check that the files in that directory are as expected:
entries = self.storage.listdir(expected_timestamped_directory)[1]
people_1_leafname = "people-000001.json"
people_2_leafname = "people-000002.json"
ballot_1_leafname = "ballots-000001.json"
ballot_2_leafname = "ballots-000002.json"
ballot_3_leafname = "ballots-000003.json"
self.assertEqual(
set(entries),
{
people_1_leafname,
people_2_leafname,
ballot_1_leafname,
ballot_2_leafname,
ballot_3_leafname,
},
)
# Get the data from those pages:
with self.storage.open(
join(expected_timestamped_directory, people_1_leafname)) as f:
persons_1_data = json.loads(f.read().decode("utf8"))
with self.storage.open(
join(expected_timestamped_directory, people_2_leafname)) as f:
persons_2_data = json.loads(f.read().decode("utf8"))
# Check the previous and next links are as we expect:
self.assertEqual(
persons_1_data["next"],
"https://example.com/media/api-cache-for-wcivf/{}/{}".format(
expected_leafname, people_2_leafname),
)
self.assertEqual(persons_1_data["previous"], None)
self.assertEqual(persons_2_data["next"], None)
self.assertEqual(
persons_2_data["previous"],
"https://example.com/media/api-cache-for-wcivf/{}/{}".format(
expected_leafname, people_1_leafname),
)
# Check that the URL of the first person is as expected,
# as well as it being the right person:
first_person = persons_1_data["results"][0]
self.assertEqual(first_person["id"], 818)
self.assertEqual(first_person["name"], "Sheila Gilmore")
self.assertEqual(
first_person["url"],
"https://candidates.democracyclub.org.uk/api/next/people/818/?format=json",
)
def _setup_cached_api_directory(self, dir_list):
"""
Saves a tmp file in settings.MEDIA_ROOT, called `.keep` in each
directory in dir_list.
"""
for d in dir_list:
self.storage.save(join("cached-api", d, ".keep"), ContentFile("."))
@patch(
"candidates.management.commands.candidates_cache_api_to_directory.datetime"
)
def test_cache_api_to_directory_prune(self, mock_datetime):
# Need to make sure datetime.strptime still works:
mock_datetime.strptime.side_effect = datetime.strptime
mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
expected_to_prune = [
"2017-05-12T08:00:00",
"2017-05-12T10:00:00",
"2017-05-12T12:00:00",
]
expected_to_keep = [
"2017-05-14T08:00:00",
"2017-05-14T10:00:00",
"2017-05-14T12:00:00",
"2017-05-14T12:33:05",
]
self._setup_cached_api_directory(expected_to_keep + expected_to_prune)
self.assertTrue(
self.storage.exists(
join("cached-api", "2017-05-12T08:00:00", ".keep")))
call_command(
"candidates_cache_api_to_directory",
page_size="3",
url_prefix="https://example.com/media/api-cache-for-wcivf/",
prune=True,
)
for dir_name in expected_to_keep:
self.assertTrue(
self.storage.exists(join("cached-api", dir_name, ".keep")))
for dir_name in expected_to_prune:
self.assertFalse(
self.storage.exists(join("cached-api", dir_name, ".keep")))
@patch(
"candidates.management.commands.candidates_cache_api_to_directory.datetime"
)
def test_cache_api_to_directory_prune_four_old(self, mock_datetime):
# Need to make sure datetime.strptime still works:
mock_datetime.strptime.side_effect = datetime.strptime
mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
expected_to_prune = ["2017-05-12T06:00:00"]
expected_to_keep = [
"2017-05-12T08:00:00",
"2017-05-12T10:00:00",
"2017-05-12T12:00:00",
"2017-05-14T12:33:05",
]
self._setup_cached_api_directory(expected_to_keep + expected_to_prune)
self.assertTrue(
self.storage.exists(
join("cached-api", "2017-05-12T06:00:00", ".keep")))
self.assertTrue(
self.storage.exists(
join("cached-api", "2017-05-12T08:00:00", ".keep")))
call_command(
"candidates_cache_api_to_directory",
page_size="3",
url_prefix="https://example.com/media/api-cache-for-wcivf/",
prune=True,
)
# Even though all of those directories are more than 36
# hours old, they should all be kept because they're the
# most recent 4:
for dir_name in expected_to_keep:
self.assertTrue(
self.storage.exists(join("cached-api", dir_name, ".keep")))
for dir_name in expected_to_prune:
self.assertFalse(
self.storage.exists(join("cached-api", dir_name, ".keep")))
def test_legacy_redirects(self):
req = self.app.get("/api/v0.9/elections/2010/")
self.assertEqual(req.status_code, 301)
self.assertEqual(req.location, "/api/v0.9/elections/parl.2010-05-06/")
req = self.app.get("/api/v0.9/elections/2010.json")
self.assertEqual(req.status_code, 301)
self.assertEqual(req.location,
"/api/v0.9/elections/parl.2010-05-06.json")
def test_legacy_contact_details(self):
person = PersonFactory()
PersonIdentifier.objects.create(value_type="twitter_username",
value="Froglet4MP",
person=person)
req = self.app.get("/api/v0.9/persons/{}/".format(person.pk))
person_json = req.json
self.assertTrue("contact_details" in person_json)
self.assertEqual(person_json["contact_details"][0]["contact_type"],
"twitter")
self.assertEqual(person_json["contact_details"][0]["value"],
"Froglet4MP")
