class StorageWrapper: def __init__(self): self.storage = DefaultStorage() def save(self, filename: str, file_like): return self.storage.save(filename, file_like) def list(self, ext: str = None): _, files = self.storage.listdir("/") if ext: files = [f for f in files if f.endswith(ext)] return files def exists(self, filename: str): return self.storage.exists(filename) def find(self, filename: str) -> Optional[str]: for file_ in self.list(): if file_ == filename: return file_ return None def write_to(self, filename: str, callback: Any): f = self.storage.open(filename) callback(f.read()) f.close() def remove(self, filename: str): self.storage.delete(filename)
def render(self, context): challenge_short_name = context.page.challenge.short_name projectpath = challenge_short_name + "/" + self.path storage = DefaultStorage() try: filenames = storage.listdir(projectpath)[1] except OSError as e: return self.make_dataset_error_msg(str(e)) filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) links = [] for filename in filenames: downloadlink = reverse( "serving:challenge-file", kwargs={ "challenge_short_name": challenge_short_name, "path": f"{self.path}/{filename}", }, ) links.append('<li><a href="' + downloadlink + '">' + filename + " </a></li>") htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>" return htmlOut
def render(self, context): challenge: Challenge = context["currentpage"].challenge try: projectpath = safe_join(challenge.get_project_data_folder(), self.path) except SuspiciousFileOperation: return self.make_dataset_error_msg( "path is outside the challenge folder.") storage = DefaultStorage() try: filenames = storage.listdir(projectpath)[1] except OSError as e: return self.make_dataset_error_msg(str(e)) filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) links = [] for filename in filenames: downloadlink = reverse( "root-serving:challenge-file", kwargs={ "challenge_name": challenge.short_name, "path": f"{self.path}/{filename}", }, ) links.append('<li><a href="' + downloadlink + '">' + filename + " </a></li>") htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>" return htmlOut
def get_dirnames(path): """ Get all directory names in path as list of strings Raises: OSError if directory can not be found """ storage = DefaultStorage() dirnames = storage.listdir(path)[0] dirnames.sort() return dirnames
def get_dirnames(path): """ Get all directory names in path as list of strings Raises: OSError if directory can not be found """ storage = DefaultStorage() dirnames = storage.listdir(path)[0] dirnames.sort() return dirnames
def cleanup_files(self): """Remove stale screenshots""" storage = DefaultStorage() try: files = storage.listdir('screenshots')[1] except OSError: return for name in files: fullname = os.path.join('screenshots', name) if not Screenshot.objects.filter(image=fullname).exists(): storage.delete(fullname)
def cleanup_screenshot_files(): """Remove stale screenshots""" storage = DefaultStorage() try: files = storage.listdir('screenshots')[1] except OSError: return for name in files: fullname = os.path.join('screenshots', name) if not Screenshot.objects.filter(image=fullname).exists(): storage.delete(fullname)
def get_filenames(self, context, path): """ Get all filenames in path Raises OSError if directory can not be found """ challenge_short_name = context.page.challenge.short_name projectpath = challenge_short_name + "/" + path storage = DefaultStorage() filenames = storage.listdir(projectpath)[1] filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) return filenames
def render(self, context): challenge: Challenge = context["currentpage"].challenge try: projectpath = safe_join( challenge.get_project_data_folder(), self.path ) except SuspiciousFileOperation: return self.make_dataset_error_msg( "path is outside the challenge folder." ) storage = DefaultStorage() try: filenames = storage.listdir(projectpath)[1] except OSError as e: return self.make_dataset_error_msg(str(e)) filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) links = [] for filename in filenames: downloadlink = reverse( "root-serving:challenge-file", kwargs={ "challenge_name": challenge.short_name, "path": f"{self.path}/{filename}", }, ) links.append( '<li><a href="' + downloadlink + '">' + filename + " </a></li>" ) htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>" return htmlOut
class TestAPI(TmpMediaRootMixin, UK2015ExamplesMixin, WebTest): def setUp(self): super().setUp() person_extra = PersonExtraFactory.create(base__id="2009", base__name="Tessa Jowell") dulwich_not_stand = PersonExtraFactory.create(base__id="4322", base__name="Helen Hayes") edinburgh_candidate = PersonExtraFactory.create( base__id="818", base__name="Sheila Gilmore") edinburgh_winner = PersonExtraFactory.create( base__id="5795", base__name="Tommy Sheppard") edinburgh_may_stand = PersonExtraFactory.create( base__id="5163", base__name="Peter McColl") MembershipFactory.create( person=person_extra.base, post=self.dulwich_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.dulwich_post_extra_pee, ) MembershipFactory.create( person=person_extra.base, organization=self.labour_party_extra.base, post_election=self.edinburgh_east_post_extra_pee, ) MembershipFactory.create( person=dulwich_not_stand.base, post=self.dulwich_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.dulwich_post_extra_pee_earlier, ) dulwich_not_stand.not_standing.add(self.election) MembershipFactory.create( person=edinburgh_winner.base, post=self.edinburgh_east_post_extra.base, on_behalf_of=self.labour_party_extra.base, elected=True, post_election=self.edinburgh_east_post_extra_pee, ) MembershipFactory.create( person=edinburgh_candidate.base, post=self.edinburgh_east_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.edinburgh_east_post_extra_pee, ) MembershipFactory.create( person=edinburgh_may_stand.base, post=self.edinburgh_east_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.edinburgh_east_post_extra_pee_earlier, ) self.storage = DefaultStorage() def test_api_basic_response(self): response = self.app.get("/api/v0.9/") self.assertEqual(response.status_code, 200) json = response.json self.assertEqual(json["persons"], "http://localhost:80/api/v0.9/persons/") self.assertEqual(json["organizations"], "http://localhost:80/api/v0.9/organizations/") self.assertEqual(json["elections"], "http://localhost:80/api/v0.9/elections/") self.assertEqual(json["posts"], "http://localhost:80/api/v0.9/posts/") persons_resp = self.app.get("/api/v0.9/persons/") self.assertEqual(persons_resp.status_code, 200) organizations_resp = self.app.get("/api/v0.9/organizations/") self.assertEqual(organizations_resp.status_code, 200) elections_resp = self.app.get("/api/v0.9/elections/") self.assertEqual(elections_resp.status_code, 200) posts_resp = self.app.get("/api/v0.9/posts/") self.assertEqual(posts_resp.status_code, 200) def test_api_errors(self): response = self.app.get("/api/", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.get("/api/v0.8", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.get("/api/v0.9/person/", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.get("/api/v0.9/persons/4000/", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.post("/api/v0.9/persons/", {}, expect_errors=True) self.assertEqual(response.status_code, 403) def test_api_persons(self): persons_resp = self.app.get("/api/v0.9/persons/") persons = persons_resp.json self.assertEqual(persons["count"], len(persons["results"])) self.assertEqual(persons["count"], 5) def test_api_person(self): person_resp = self.app.get("/api/v0.9/persons/2009/") self.assertEqual(person_resp.status_code, 200) person = person_resp.json self.assertEqual(person["id"], 2009) self.assertEqual(person["name"], "Tessa Jowell") memberships = sorted(person["memberships"], key=lambda m: m["role"]) self.assertEqual(len(memberships), 2) self.assertEqual(memberships[1]["role"], "Candidate") self.assertEqual(len(person["versions"]), 0) def test_api_organizations(self): organizations_resp = self.app.get("/api/v0.9/organizations/") organizations = organizations_resp.json self.assertEqual(organizations["count"], len(organizations["results"])) self.assertEqual(organizations["count"], 7) def test_api_organization(self): organizations_resp = self.app.get("/api/v0.9/organizations/") organizations = organizations_resp.json organization_url = None for organization in organizations["results"]: if organization["id"] == "party:53": organization_url = organization["url"] break organization_resp = self.app.get(organization_url) self.assertEqual(organization_resp.status_code, 200) organization = organization_resp.json self.assertEqual(organization["id"], "party:53") self.assertEqual(organization["name"], "Labour Party") def test_api_elections(self): elections_resp = self.app.get("/api/v0.9/elections/") elections = elections_resp.json self.assertEqual(elections["count"], len(elections["results"])) self.assertEqual(elections["count"], 3) def test_api_election(self): elections_resp = self.app.get("/api/v0.9/elections/") elections = elections_resp.json election_url = None for election in elections["results"]: if election["id"] == "2015": election_url = election["url"] break election_resp = self.app.get(election_url) self.assertEqual(election_resp.status_code, 200) election = election_resp.json self.assertEqual(election["id"], "2015") self.assertEqual(election["name"], "2015 General Election") def test_api_posts(self): posts_resp = self.app.get("/api/v0.9/posts/") posts = posts_resp.json self.assertEqual(posts["count"], len(posts["results"])) self.assertEqual(posts["count"], 5) def test_api_post(self): posts_resp = self.app.get("/api/v0.9/posts/") posts = posts_resp.json post_url = None for post in posts["results"]: if post["id"] == "65808": post_url = post["url"] break self.assertTrue(post_url) post_resp = self.app.get(post_url) self.assertEqual(post_resp.status_code, 200) post = post_resp.json self.assertEqual(post["id"], "65808") self.assertEqual(post["label"], "Member of Parliament for Dulwich and West Norwood") def test_api_person_redirects(self): PersonRedirect.objects.create(old_person_id="1234", new_person_id="42") PersonRedirect.objects.create(old_person_id="5678", new_person_id="12") person_redirects_resp = self.app.get("/api/v0.9/person_redirects/") person_redirects = person_redirects_resp.json self.assertEqual(person_redirects["results"][0]["old_person_id"], 1234) self.assertEqual(person_redirects["results"][0]["new_person_id"], 42) self.assertEqual(person_redirects["results"][1]["old_person_id"], 5678) self.assertEqual(person_redirects["results"][1]["new_person_id"], 12) def test_api_person_redirect(self): PersonRedirect.objects.create(old_person_id="1234", new_person_id="42") url = "/api/v0.9/person_redirects/1234/" person_redirect_resp = self.app.get(url) person_redirect = person_redirect_resp.json self.assertEqual(person_redirect["old_person_id"], 1234) self.assertEqual(person_redirect["new_person_id"], 42) def test_api_version_info(self): version_resp = self.app.get("/version.json") self.assertEqual(version_resp.status_code, 200) info = version_resp.json self.assertEqual(info["users_who_have_edited"], 0) self.assertEqual(info["interesting_user_actions"], 0) LoggedAction.objects.create(action_type="set-candidate-not-elected") LoggedAction.objects.create(action_type="edit-candidate") version_resp = self.app.get("/version.json") info = version_resp.json self.assertEqual(info["interesting_user_actions"], 1) def test_api_cors_headers(self): resp = self.app.get("/api/v0.9/", headers={"Origin": b"http://example.com"}) self.assertTrue("Access-Control-Allow-Origin" in resp.headers) self.assertEqual(resp.headers["Access-Control-Allow-Origin"], "*") resp = self.app.get("/", headers={"Origin": b"http://example.com"}) self.assertFalse("Access-Control-Allow-Origin" in resp.headers) def test_api_jsonp_response(self): response = self.app.get("/api/v0.9/?format=jsonp&callback=test") self.assertEqual(response.status_code, 200) self.assertTrue(response.text.startswith("test(")) @patch( "candidates.management.commands.candidates_cache_api_to_directory.datetime" ) def test_persons_api_to_directory(self, mock_datetime): # current # timestamped # timestamped mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0) target_directory = settings.MEDIA_ROOT call_command( "candidates_cache_api_to_directory", page_size="3", url_prefix="https://example.com/media/api-cache-for-wcivf/", ) expected_leafname = "2017-05-14T12:33:05" expected_timestamped_directory = join(settings.MEDIA_ROOT, "cached-api", expected_leafname) expected_path = join("cached-api", "latest") self.assertTrue(self.storage.exists(expected_timestamped_directory)) self.assertTrue(self.storage.exists(expected_path)) # Check that the files in that directory are as expected: entries = self.storage.listdir(expected_timestamped_directory)[1] persons_1_leafname = "persons-000001.json" persons_2_leafname = "persons-000002.json" posts_1_leafname = "posts-000001.json" posts_2_leafname = "posts-000002.json" self.assertEqual( set(entries), { persons_1_leafname, persons_2_leafname, posts_1_leafname, posts_2_leafname, }, ) # Get the data from those pages: with self.storage.open( join(expected_timestamped_directory, persons_1_leafname)) as f: persons_1_data = json.loads(f.read().decode('utf8')) with self.storage.open( join(expected_timestamped_directory, persons_2_leafname)) as f: persons_2_data = json.loads(f.read().decode('utf8')) with self.storage.open( join(expected_timestamped_directory, posts_1_leafname)) as f: posts_1_data = json.loads(f.read().decode('utf8')) with self.storage.open( join(expected_timestamped_directory, posts_2_leafname)) as f: posts_2_data = json.loads(f.read().decode('utf8')) # Check the previous and next links are as we expect: self.assertEqual( persons_1_data["next"], "https://example.com/media/api-cache-for-wcivf/{}/{}".format( expected_leafname, persons_2_leafname), ) self.assertEqual(persons_1_data["previous"], None) self.assertEqual(persons_2_data["next"], None) self.assertEqual( persons_2_data["previous"], "https://example.com/media/api-cache-for-wcivf/{}/{}".format( expected_leafname, persons_1_leafname), ) self.assertEqual( posts_1_data["next"], "https://example.com/media/api-cache-for-wcivf/{}/{}".format( expected_leafname, posts_2_leafname), ) self.assertEqual(posts_1_data["previous"], None) self.assertEqual(posts_2_data["next"], None) self.assertEqual( posts_2_data["previous"], "https://example.com/media/api-cache-for-wcivf/{}/{}".format( expected_leafname, posts_1_leafname), ) # Check that the URL of the first person is as expected, # as well as it being the right person: first_person = persons_1_data["results"][0] self.assertEqual(first_person["id"], 818) self.assertEqual(first_person["name"], "Sheila Gilmore") self.assertEqual( first_person["url"], "https://candidates.democracyclub.org.uk/api/v0.9/persons/818/?format=json", ) # Similarly, check that the URL of the first post is as expected: first_post = posts_1_data["results"][0] self.assertEqual(first_post["id"], self.edinburgh_east_post_extra.slug) self.assertEqual(first_post["label"], "Member of Parliament for Edinburgh East") self.assertEqual( first_post["url"], "https://candidates.democracyclub.org.uk/api/v0.9/posts/14419/?format=json", ) def _setup_cached_api_directory(self, dir_list): """ Saves a tmp file in settings.MEDIA_ROOT, called `.keep` in each directory in dir_list. """ for d in dir_list: self.storage.save(join("cached-api", d, ".keep"), ContentFile(".")) @patch( "candidates.management.commands.candidates_cache_api_to_directory.datetime" ) def test_cache_api_to_directory_prune(self, mock_datetime): # Need to make sure datetime.strptime still works: mock_datetime.strptime.side_effect = datetime.strptime mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0) expected_to_prune = [ "2017-05-12T08:00:00", "2017-05-12T10:00:00", "2017-05-12T12:00:00", ] expected_to_keep = [ "2017-05-14T08:00:00", "2017-05-14T10:00:00", "2017-05-14T12:00:00", "2017-05-14T12:33:05", ] self._setup_cached_api_directory(expected_to_keep + expected_to_prune) self.assertTrue( self.storage.exists( join("cached-api", "2017-05-12T08:00:00", ".keep"))) call_command( "candidates_cache_api_to_directory", page_size="3", url_prefix="https://example.com/media/api-cache-for-wcivf/", prune=True, ) for dir_name in expected_to_keep: self.assertTrue( self.storage.exists(join("cached-api", dir_name, ".keep"))) for dir_name in expected_to_prune: self.assertFalse( self.storage.exists(join("cached-api", dir_name, ".keep"))) @patch( "candidates.management.commands.candidates_cache_api_to_directory.datetime" ) def test_cache_api_to_directory_prune_four_old(self, mock_datetime): # Need to make sure datetime.strptime still works: mock_datetime.strptime.side_effect = datetime.strptime mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0) expected_to_prune = ["2017-05-12T06:00:00"] expected_to_keep = [ "2017-05-12T08:00:00", "2017-05-12T10:00:00", "2017-05-12T12:00:00", "2017-05-14T12:33:05", ] self._setup_cached_api_directory(expected_to_keep + expected_to_prune) self.assertTrue( self.storage.exists( join("cached-api", "2017-05-12T06:00:00", ".keep"))) self.assertTrue( self.storage.exists( join("cached-api", "2017-05-12T08:00:00", ".keep"))) call_command( "candidates_cache_api_to_directory", page_size="3", url_prefix="https://example.com/media/api-cache-for-wcivf/", prune=True, ) # Even though all of those directories are more than 36 # hours old, they should all be kept because they're the # most recent 4: for dir_name in expected_to_keep: self.assertTrue( self.storage.exists(join("cached-api", dir_name, ".keep"))) for dir_name in expected_to_prune: self.assertFalse( self.storage.exists(join("cached-api", dir_name, ".keep")))
class CSVTests(TmpMediaRootMixin, TestUserMixin, UK2015ExamplesMixin, TestCase): def setUp(self): super().setUp() self.storage = DefaultStorage() # The second person's name (and party name) have diacritics in # them to test handling of Unicode when outputting to CSV. self.gb_person = people.tests.factories.PersonFactory.create( id=2009, name="Tessa Jowell", honorific_suffix="DBE", honorific_prefix="Ms", gender="female", ) PersonImage.objects.create_from_file( EXAMPLE_IMAGE_FILENAME, "images/jowell-pilot.jpg", defaults={ "person": self.gb_person, "is_primary": True, "source": "Taken from Wikipedia", "copyright": "example-license", "uploading_user": self.user, "user_notes": "A photo of Tessa Jowell", }, ) self.ni_person = people.tests.factories.PersonFactory.create( id=1953, name="Daithí McKay", gender="male" ) north_antrim_post = factories.PostFactory.create( elections=(self.election, self.earlier_election), organization=self.commons, slug="66135", label="Member of Parliament for North Antrim", party_set=self.ni_parties, ) factories.MembershipFactory.create( person=self.ni_person, post=north_antrim_post, party=self.sinn_fein, post_election=self.election.postextraelection_set.get( post=north_antrim_post ), ) factories.MembershipFactory.create( person=self.ni_person, post=north_antrim_post, party=self.sinn_fein, post_election=self.earlier_election.postextraelection_set.get( post=north_antrim_post ), ) factories.MembershipFactory.create( person=self.gb_person, post=self.camberwell_post, party=self.labour_party, post_election=self.camberwell_post_pee, ) factories.MembershipFactory.create( person=self.gb_person, post=self.dulwich_post, party=self.labour_party, post_election=self.dulwich_post_pee_earlier, ) self.gb_person.tmp_person_identifiers.create( internal_identifier="10326", value_type="theyworkforyou" ) self.gb_person.tmp_person_identifiers.create( value_type="email", value="*****@*****.**" ) def test_as_single_dict(self): membership = ( Membership.objects.for_csv() .filter(person=self.gb_person.id) .first() ) # After the select_related and prefetch_related calls # PersonExtra there should only be one more query - that to # find the complex fields mapping: with self.assertNumQueries(0): membership_dict = membership.dict_for_csv() self.assertEqual( sorted(list(membership_dict.keys())), sorted(settings.CSV_ROW_FIELDS), ) self.assertEqual(len(membership_dict.keys()), 39) self.assertEqual(membership_dict["id"], 2009) self.assertEqual( membership_dict["parlparse_id"], "uk.org.publicwhip/person/10326" ) self.assertEqual( membership_dict["theyworkforyou_url"], "http://www.theyworkforyou.com/mp/10326", ) def test_as_dict_2010(self): with self.assertNumQueries(5): memberships_dicts, elected = memberships_dicts_for_csv( self.earlier_election.slug ) self.assertEqual(len(memberships_dicts[self.earlier_election.slug]), 2) membership_dict = memberships_dicts["parl.2010-05-06"][1] self.assertEqual(len(membership_dict.keys()), 39) self.assertEqual(membership_dict["id"], 2009) def test_csv_output(self): tessa_image_url = self.gb_person.primary_image.url d = { "election_date": date_in_near_future, "earlier_election_date": date_in_near_future - timedelta(days=FOUR_YEARS_IN_DAYS), } PersonRedirect.objects.create(old_person_id=12, new_person_id=1953) PersonRedirect.objects.create(old_person_id=56, new_person_id=1953) self.maxDiff = None example_output = ( "id,name,honorific_prefix,honorific_suffix,gender,birth_date,election,party_id,party_name,post_id,post_label,mapit_url,elected,email,twitter_username,facebook_page_url,party_ppc_page_url,facebook_personal_url,homepage_url,wikipedia_url,linkedin_url,image_url,proxy_image_url_template,image_copyright,image_uploading_user,image_uploading_user_notes,twitter_user_id,election_date,election_current,party_lists_in_use,party_list_position,old_person_ids,gss_code,parlparse_id,theyworkforyou_url,party_ec_id,favourite_biscuits,cancelled_poll,wikidata_url\r\n" + "1953,Daith\xed McKay,,,male,,parl.2010-05-06,party:39,Sinn F\xe9in,66135,North Antrim,,,,,,,,,,,,,,,,,{earlier_election_date},False,False,,12;56,,,,PP39,,False,\r\n".format( **d ) + "2009,Tessa Jowell,Ms,DBE,female,,parl.2010-05-06,party:53,Labour Party,65808,Dulwich and West Norwood,,,[email protected],,,,,,,,{image_url},,example-license,john,A photo of Tessa Jowell,,{earlier_election_date},False,False,,,,uk.org.publicwhip/person/10326,http://www.theyworkforyou.com/mp/10326,PP53,,False,\r\n".format( image_url=tessa_image_url, **d ) + "1953,Daith\xed McKay,,,male,,parl.2015-05-07,party:39,Sinn F\xe9in,66135,North Antrim,,,,,,,,,,,,,,,,,{election_date},True,False,,12;56,,,,PP39,,False,\r\n".format( **d ) + "2009,Tessa Jowell,Ms,DBE,female,,parl.2015-05-07,party:53,Labour Party,65913,Camberwell and Peckham,,,[email protected],,,,,,,,{image_url},,example-license,john,A photo of Tessa Jowell,,{election_date},True,False,,,,uk.org.publicwhip/person/10326,http://www.theyworkforyou.com/mp/10326,PP53,,False,\r\n".format( image_url=tessa_image_url, **d ) ) with self.assertNumQueries(5): memberships_dicts, elected = memberships_dicts_for_csv() all_members = [] for slug, members in memberships_dicts.items(): all_members += members self.assertEqual(list_to_csv(all_members), example_output) def test_create_csv_management_command(self): # An empty media directory self.assertEqual( self.storage.listdir(self.storage.base_location)[1], [] ) # We expect a CSV file per election, and one for all elections call_command("candidates_create_csv") self.assertEqual( set(sorted(self.storage.listdir(".")[1])), set( [ "candidates-parl.2010-05-06.csv", "candidates-parl.2015-05-07.csv", "candidates-all.csv", "candidates-elected-all.csv", "candidates-local.maidstone.2016-05-05.csv", "candidates-2015-05-07.csv", "candidates-2010-05-06.csv", ] ), ) def test_create_csv_management_command_single_election(self): # An empty media directory self.assertEqual( self.storage.listdir(self.storage.base_location), (["images"], []) ) # We expect a single CSV file call_command("candidates_create_csv", "--election", "parl.2015-05-07") self.assertSetEqual( set(self.storage.listdir(".")[1]), set( [ "candidates-parl.2010-05-06.csv", "candidates-parl.2015-05-07.csv", "candidates-local.maidstone.2016-05-05.csv", "candidates-2015-05-07.csv", ] ), ) def test_create_csv_management_command_single_election_doesnt_exist(self): # An empty media directory, apart from the images dir self.assertEqual( self.storage.listdir(self.storage.base_location), (["images"], []) ) with self.assertRaises(CommandError): call_command("candidates_create_csv", "--election", "foo") def test_create_csv_no_candidates_for_election(self): # An empty media directory, apart from the images dir self.assertEqual( self.storage.listdir(self.storage.base_location), (["images"], []) ) factories.ElectionFactory.create( slug="2018", name="2018 General Election", for_post_role="Member of Parliament", organization=self.commons, ) call_command("candidates_create_csv") self.assertSetEqual( set(self.storage.listdir(".")[1]), set( [ "candidates-parl.2010-05-06.csv", "candidates-parl.2015-05-07.csv", "candidates-2018.csv", "candidates-all.csv", "candidates-elected-all.csv", "candidates-local.maidstone.2016-05-05.csv", "candidates-2015-05-07.csv", "candidates-2010-05-06.csv", ] ), ) empty_file = self.storage.open("candidates-2018.csv").read() self.assertEqual(len(empty_file.splitlines()), 1) self.assertEqual( empty_file.splitlines()[0].decode(), (",".join(settings.CSV_ROW_FIELDS)), ) non_empty_file = self.storage.open( "candidates-parl.2015-05-07.csv" ).read() self.assertEqual(len(non_empty_file.splitlines()), 3)
class TestAPI(TestUserMixin, TmpMediaRootMixin, UK2015ExamplesMixin, WebTest): def setUp(self): super().setUp() person = PersonFactory.create(id=2009, name="Tessa Jowell") PersonImage.objects.update_or_create_from_file( EXAMPLE_IMAGE_FILENAME, "images/imported.jpg", person, defaults={ "md5sum": "md5sum", "copyright": "example-license", "uploading_user": self.user, "user_notes": "Here's an image...", "is_primary": True, "source": "Found on the candidate's Flickr feed", }, ) dulwich_not_stand = PersonFactory.create(id=4322, name="Helen Hayes") edinburgh_candidate = PersonFactory.create(id="818", name="Sheila Gilmore") edinburgh_winner = PersonFactory.create(id="5795", name="Tommy Sheppard") edinburgh_may_stand = PersonFactory.create(id="5163", name="Peter McColl") MembershipFactory.create( person=person, post=self.dulwich_post, party=self.labour_party, ballot=self.dulwich_post_ballot, ) MembershipFactory.create(person=person, ballot=self.edinburgh_east_post_ballot) MembershipFactory.create( person=dulwich_not_stand, post=self.dulwich_post, party=self.labour_party, ballot=self.dulwich_post_ballot_earlier, ) dulwich_not_stand.not_standing.add(self.election) MembershipFactory.create( person=edinburgh_winner, post=self.edinburgh_east_post, party=self.labour_party, elected=True, ballot=self.edinburgh_east_post_ballot, ) MembershipFactory.create( person=edinburgh_candidate, post=self.edinburgh_east_post, party=self.labour_party, ballot=self.edinburgh_east_post_ballot, ) MembershipFactory.create( person=edinburgh_may_stand, post=self.edinburgh_east_post, party=self.labour_party, ballot=self.edinburgh_east_post_ballot_earlier, ) self.storage = DefaultStorage() def test_api_basic_response(self): response = self.app.get("/api/v0.9/") self.assertEqual(response.status_code, 200) json = response.json self.assertEqual(json["persons"], "http://testserver/api/v0.9/persons/") self.assertEqual(json["organizations"], "http://testserver/api/v0.9/organizations/") self.assertEqual(json["elections"], "http://testserver/api/v0.9/elections/") self.assertEqual(json["posts"], "http://testserver/api/v0.9/posts/") persons_resp = self.app.get("/api/v0.9/persons/") self.assertEqual(persons_resp.status_code, 200) organizations_resp = self.app.get("/api/v0.9/organizations/") self.assertEqual(organizations_resp.status_code, 200) elections_resp = self.app.get("/api/v0.9/elections/") self.assertEqual(elections_resp.status_code, 200) posts_resp = self.app.get("/api/v0.9/posts/") self.assertEqual(posts_resp.status_code, 200) def test_api_home(self): response = self.app.get("/api/") self.assertEqual(response.status_code, 302) response = self.app.get("/api/docs/") self.assertEqual(response.status_code, 200) response = self.app.get("/api/v0.8", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.get("/api/v0.9/person/", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.get("/api/v0.9/persons/4000/", expect_errors=True) self.assertEqual(response.status_code, 404) response = self.app.post("/api/v0.9/persons/", params={}, expect_errors=True) self.assertEqual(response.status_code, 403) def test_api_persons(self): persons_resp = self.app.get("/api/v0.9/persons/") persons = persons_resp.json self.assertEqual(persons["count"], len(persons["results"])) self.assertEqual(persons["count"], 5) def test_api_person(self): person_resp = self.app.get("/api/v0.9/persons/2009/") self.assertEqual(person_resp.status_code, 200) person = person_resp.json self.assertEqual(person["id"], 2009) self.assertEqual(person["name"], "Tessa Jowell") memberships = sorted(person["memberships"], key=lambda m: m["role"]) self.assertEqual(len(memberships), 2) self.assertEqual(memberships[1]["role"], "Candidate") self.assertEqual(len(person["versions"]), 0) def _make_legacy_parties(self): """ It used to be that political parties were stored on the "Organization" model. for maintaining v0.9 API compatibility we've not deleted them from that model (yet), so let's make the test data support this legacy case """ from candidates.tests.factories import OrganizationFactory from candidates.tests.uk_examples import EXAMPLE_PARTIES for party in EXAMPLE_PARTIES: p = OrganizationFactory(slug=party["legacy_slug"], name=party["name"]) def test_api_legacy_organizations_with_parties(self): self._make_legacy_parties() organizations_resp = self.app.get("/api/v0.9/organizations/") organizations = organizations_resp.json self.assertEqual(organizations["count"], len(organizations["results"])) self.assertEqual(organizations["count"], 7) def test_api_legacy_organization_with_parties(self): self._make_legacy_parties() organizations_resp = self.app.get("/api/v0.9/organizations/") organizations = organizations_resp.json organization_url = None for organization in organizations["results"]: if organization["id"] == "party:53": organization_url = organization["url"] break organization_resp = self.app.get(organization_url) self.assertEqual(organization_resp.status_code, 200) organization = organization_resp.json self.assertEqual(organization["id"], "party:53") self.assertEqual(organization["name"], "Labour Party") def test_api_elections(self): elections_resp = self.app.get("/api/v0.9/elections/") elections = elections_resp.json self.assertEqual(elections["count"], len(elections["results"])) self.assertEqual(elections["count"], 3) def test_api_elections_without_orgs(self): # Regression test that we can serve elections without an organzation self.election.organization = None self.election.save() elections_resp = self.app.get("/api/v0.9/elections/", expect_errors=True) self.assertEqual(elections_resp.status_code, 200) def test_api_election(self): elections_resp = self.app.get("/api/v0.9/elections/") elections = elections_resp.json election_url = None for election in elections["results"]: if election["id"] == "parl.2015-05-07": election_url = election["url"] break election_resp = self.app.get(election_url) self.assertEqual(election_resp.status_code, 200) election = election_resp.json self.assertEqual(election["id"], "parl.2015-05-07") self.assertEqual(election["name"], "2015 General Election") def test_api_posts(self): posts_resp = self.app.get("/api/v0.9/posts/") posts = posts_resp.json self.assertEqual(posts["count"], len(posts["results"])) self.assertEqual(posts["count"], 5) def test_api_post(self): posts_resp = self.app.get("/api/v0.9/posts/") posts = posts_resp.json post_url = None for post in posts["results"]: if post["id"] == "65808": post_url = post["url"] break self.assertTrue(post_url) post_resp = self.app.get(post_url) self.assertEqual(post_resp.status_code, 200) post = post_resp.json self.assertEqual(post["id"], "65808") self.assertEqual(post["label"], "Member of Parliament for Dulwich and West Norwood") def test_api_person_redirects(self): PersonRedirect.objects.create(old_person_id="1234", new_person_id="42") PersonRedirect.objects.create(old_person_id="5678", new_person_id="12") person_redirects_resp = self.app.get("/api/v0.9/person_redirects/") person_redirects = person_redirects_resp.json self.assertEqual(person_redirects["results"][0]["old_person_id"], 1234) self.assertEqual(person_redirects["results"][0]["new_person_id"], 42) self.assertEqual(person_redirects["results"][1]["old_person_id"], 5678) self.assertEqual(person_redirects["results"][1]["new_person_id"], 12) def test_api_person_redirect(self): PersonRedirect.objects.create(old_person_id="1234", new_person_id="42") url = "/api/v0.9/person_redirects/1234/" person_redirect_resp = self.app.get(url) person_redirect = person_redirect_resp.json self.assertEqual(person_redirect["old_person_id"], 1234) self.assertEqual(person_redirect["new_person_id"], 42) def test_api_version_info(self): version_resp = self.app.get("/version.json") self.assertEqual(version_resp.status_code, 200) info = version_resp.json self.assertEqual(info["users_who_have_edited"], 0) self.assertEqual(info["interesting_user_actions"], 0) LoggedAction.objects.create(action_type="set-candidate-not-elected") LoggedAction.objects.create(action_type="edit-candidate") version_resp = self.app.get("/version.json") info = version_resp.json self.assertEqual(info["interesting_user_actions"], 1) def test_api_cors_headers(self): resp = self.app.get("/api/v0.9/", headers={"Origin": b"http://example.com"}) self.assertTrue("Access-Control-Allow-Origin" in resp.headers) self.assertEqual(resp.headers["Access-Control-Allow-Origin"], "*") resp = self.app.get("/", headers={"Origin": b"http://example.com"}) self.assertFalse("Access-Control-Allow-Origin" in resp.headers) def test_api_jsonp_response(self): response = self.app.get("/api/v0.9/?format=jsonp&callback=test") self.assertEqual(response.status_code, 200) self.assertTrue(response.text.startswith("test(")) @patch( "candidates.management.commands.candidates_cache_api_to_directory.datetime" ) def test_persons_api_to_directory(self, mock_datetime): # current # timestamped # timestamped mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0) target_directory = settings.MEDIA_ROOT call_command( "candidates_cache_api_to_directory", page_size="3", url_prefix="https://example.com/media/api-cache-for-wcivf", ) expected_leafname = "2017-05-14T12:33:05" expected_timestamped_directory = join(settings.MEDIA_ROOT, "cached-api", expected_leafname) expected_path = join("cached-api", "latest") self.assertTrue(self.storage.exists(expected_timestamped_directory)) self.assertTrue(self.storage.exists(expected_path)) # Check that the files in that directory are as expected: entries = self.storage.listdir(expected_timestamped_directory)[1] people_1_leafname = "people-000001.json" people_2_leafname = "people-000002.json" ballot_1_leafname = "ballots-000001.json" ballot_2_leafname = "ballots-000002.json" ballot_3_leafname = "ballots-000003.json" self.assertEqual( set(entries), { people_1_leafname, people_2_leafname, ballot_1_leafname, ballot_2_leafname, ballot_3_leafname, }, ) # Get the data from those pages: with self.storage.open( join(expected_timestamped_directory, people_1_leafname)) as f: persons_1_data = json.loads(f.read().decode("utf8")) with self.storage.open( join(expected_timestamped_directory, people_2_leafname)) as f: persons_2_data = json.loads(f.read().decode("utf8")) # Check the previous and next links are as we expect: self.assertEqual( persons_1_data["next"], "https://example.com/media/api-cache-for-wcivf/{}/{}".format( expected_leafname, people_2_leafname), ) self.assertEqual(persons_1_data["previous"], None) self.assertEqual(persons_2_data["next"], None) self.assertEqual( persons_2_data["previous"], "https://example.com/media/api-cache-for-wcivf/{}/{}".format( expected_leafname, people_1_leafname), ) # Check that the URL of the first person is as expected, # as well as it being the right person: first_person = persons_1_data["results"][0] self.assertEqual(first_person["id"], 818) self.assertEqual(first_person["name"], "Sheila Gilmore") self.assertEqual( first_person["url"], "https://candidates.democracyclub.org.uk/api/next/people/818/?format=json", ) def _setup_cached_api_directory(self, dir_list): """ Saves a tmp file in settings.MEDIA_ROOT, called `.keep` in each directory in dir_list. """ for d in dir_list: self.storage.save(join("cached-api", d, ".keep"), ContentFile(".")) @patch( "candidates.management.commands.candidates_cache_api_to_directory.datetime" ) def test_cache_api_to_directory_prune(self, mock_datetime): # Need to make sure datetime.strptime still works: mock_datetime.strptime.side_effect = datetime.strptime mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0) expected_to_prune = [ "2017-05-12T08:00:00", "2017-05-12T10:00:00", "2017-05-12T12:00:00", ] expected_to_keep = [ "2017-05-14T08:00:00", "2017-05-14T10:00:00", "2017-05-14T12:00:00", "2017-05-14T12:33:05", ] self._setup_cached_api_directory(expected_to_keep + expected_to_prune) self.assertTrue( self.storage.exists( join("cached-api", "2017-05-12T08:00:00", ".keep"))) call_command( "candidates_cache_api_to_directory", page_size="3", url_prefix="https://example.com/media/api-cache-for-wcivf/", prune=True, ) for dir_name in expected_to_keep: self.assertTrue( self.storage.exists(join("cached-api", dir_name, ".keep"))) for dir_name in expected_to_prune: self.assertFalse( self.storage.exists(join("cached-api", dir_name, ".keep"))) @patch( "candidates.management.commands.candidates_cache_api_to_directory.datetime" ) def test_cache_api_to_directory_prune_four_old(self, mock_datetime): # Need to make sure datetime.strptime still works: mock_datetime.strptime.side_effect = datetime.strptime mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0) expected_to_prune = ["2017-05-12T06:00:00"] expected_to_keep = [ "2017-05-12T08:00:00", "2017-05-12T10:00:00", "2017-05-12T12:00:00", "2017-05-14T12:33:05", ] self._setup_cached_api_directory(expected_to_keep + expected_to_prune) self.assertTrue( self.storage.exists( join("cached-api", "2017-05-12T06:00:00", ".keep"))) self.assertTrue( self.storage.exists( join("cached-api", "2017-05-12T08:00:00", ".keep"))) call_command( "candidates_cache_api_to_directory", page_size="3", url_prefix="https://example.com/media/api-cache-for-wcivf/", prune=True, ) # Even though all of those directories are more than 36 # hours old, they should all be kept because they're the # most recent 4: for dir_name in expected_to_keep: self.assertTrue( self.storage.exists(join("cached-api", dir_name, ".keep"))) for dir_name in expected_to_prune: self.assertFalse( self.storage.exists(join("cached-api", dir_name, ".keep"))) def test_legacy_redirects(self): req = self.app.get("/api/v0.9/elections/2010/") self.assertEqual(req.status_code, 301) self.assertEqual(req.location, "/api/v0.9/elections/parl.2010-05-06/") req = self.app.get("/api/v0.9/elections/2010.json") self.assertEqual(req.status_code, 301) self.assertEqual(req.location, "/api/v0.9/elections/parl.2010-05-06.json") def test_legacy_contact_details(self): person = PersonFactory() PersonIdentifier.objects.create(value_type="twitter_username", value="Froglet4MP", person=person) req = self.app.get("/api/v0.9/persons/{}/".format(person.pk)) person_json = req.json self.assertTrue("contact_details" in person_json) self.assertEqual(person_json["contact_details"][0]["contact_type"], "twitter") self.assertEqual(person_json["contact_details"][0]["value"], "Froglet4MP")