Пример #1
0
class StorageWrapper:
    def __init__(self):
        self.storage = DefaultStorage()

    def save(self, filename: str, file_like):
        return self.storage.save(filename, file_like)

    def list(self, ext: str = None):
        _, files = self.storage.listdir("/")
        if ext:
            files = [f for f in files if f.endswith(ext)]
        return files

    def exists(self, filename: str):
        return self.storage.exists(filename)

    def find(self, filename: str) -> Optional[str]:
        for file_ in self.list():
            if file_ == filename:
                return file_
        return None

    def write_to(self, filename: str, callback: Any):
        f = self.storage.open(filename)
        callback(f.read())
        f.close()

    def remove(self, filename: str):
        self.storage.delete(filename)
Пример #2
0
    def render(self, context):
        challenge_short_name = context.page.challenge.short_name
        projectpath = challenge_short_name + "/" + self.path
        storage = DefaultStorage()
        try:
            filenames = storage.listdir(projectpath)[1]
        except OSError as e:
            return self.make_dataset_error_msg(str(e))

        filenames.sort()
        # if extensionsFilter is given,  show only filenames with those extensions
        if "extensionFilter" in self.args.keys():
            extensions = self.args["extensionFilter"].split(",")
            filenames = filter_by_extension(filenames, extensions)
        links = []
        for filename in filenames:
            downloadlink = reverse(
                "serving:challenge-file",
                kwargs={
                    "challenge_short_name": challenge_short_name,
                    "path": f"{self.path}/{filename}",
                },
            )
            links.append('<li><a href="' + downloadlink + '">' + filename +
                         " </a></li>")
        htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>"
        return htmlOut
Пример #3
0
    def render(self, context):
        challenge: Challenge = context["currentpage"].challenge

        try:
            projectpath = safe_join(challenge.get_project_data_folder(),
                                    self.path)
        except SuspiciousFileOperation:
            return self.make_dataset_error_msg(
                "path is outside the challenge folder.")

        storage = DefaultStorage()
        try:
            filenames = storage.listdir(projectpath)[1]
        except OSError as e:
            return self.make_dataset_error_msg(str(e))

        filenames.sort()
        # if extensionsFilter is given,  show only filenames with those extensions
        if "extensionFilter" in self.args.keys():
            extensions = self.args["extensionFilter"].split(",")
            filenames = filter_by_extension(filenames, extensions)
        links = []
        for filename in filenames:
            downloadlink = reverse(
                "root-serving:challenge-file",
                kwargs={
                    "challenge_name": challenge.short_name,
                    "path": f"{self.path}/{filename}",
                },
            )
            links.append('<li><a href="' + downloadlink + '">' + filename +
                         " </a></li>")
        htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>"
        return htmlOut
Пример #4
0
def get_dirnames(path):
    """ Get all directory names in path as list of strings
            
    Raises: OSError if directory can not be found
    """
    storage = DefaultStorage()
    dirnames = storage.listdir(path)[0]
    dirnames.sort()
    return dirnames
Пример #5
0
def get_dirnames(path):
    """ Get all directory names in path as list of strings
            
    Raises: OSError if directory can not be found
    """
    storage = DefaultStorage()
    dirnames = storage.listdir(path)[0]
    dirnames.sort()
    return dirnames
Пример #6
0
 def cleanup_files(self):
     """Remove stale screenshots"""
     storage = DefaultStorage()
     try:
         files = storage.listdir('screenshots')[1]
     except OSError:
         return
     for name in files:
         fullname = os.path.join('screenshots', name)
         if not Screenshot.objects.filter(image=fullname).exists():
             storage.delete(fullname)
Пример #7
0
def cleanup_screenshot_files():
    """Remove stale screenshots"""
    storage = DefaultStorage()
    try:
        files = storage.listdir('screenshots')[1]
    except OSError:
        return
    for name in files:
        fullname = os.path.join('screenshots', name)
        if not Screenshot.objects.filter(image=fullname).exists():
            storage.delete(fullname)
Пример #8
0
 def get_filenames(self, context, path):
     """ Get all filenames in path
     
     Raises OSError if directory can not be found
     """
     challenge_short_name = context.page.challenge.short_name
     projectpath = challenge_short_name + "/" + path
     storage = DefaultStorage()
     filenames = storage.listdir(projectpath)[1]
     filenames.sort()
     # if extensionsFilter is given,  show only filenames with those extensions
     if "extensionFilter" in self.args.keys():
         extensions = self.args["extensionFilter"].split(",")
         filenames = filter_by_extension(filenames, extensions)
     return filenames
Пример #9
0
    def render(self, context):
        challenge: Challenge = context["currentpage"].challenge

        try:
            projectpath = safe_join(
                challenge.get_project_data_folder(), self.path
            )
        except SuspiciousFileOperation:
            return self.make_dataset_error_msg(
                "path is outside the challenge folder."
            )

        storage = DefaultStorage()
        try:
            filenames = storage.listdir(projectpath)[1]
        except OSError as e:
            return self.make_dataset_error_msg(str(e))

        filenames.sort()
        # if extensionsFilter is given,  show only filenames with those extensions
        if "extensionFilter" in self.args.keys():
            extensions = self.args["extensionFilter"].split(",")
            filenames = filter_by_extension(filenames, extensions)
        links = []
        for filename in filenames:
            downloadlink = reverse(
                "root-serving:challenge-file",
                kwargs={
                    "challenge_name": challenge.short_name,
                    "path": f"{self.path}/{filename}",
                },
            )
            links.append(
                '<li><a href="' + downloadlink + '">' + filename + " </a></li>"
            )
        htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>"
        return htmlOut
Пример #10
0
class TestAPI(TmpMediaRootMixin, UK2015ExamplesMixin, WebTest):
    def setUp(self):
        super().setUp()

        person_extra = PersonExtraFactory.create(base__id="2009",
                                                 base__name="Tessa Jowell")
        dulwich_not_stand = PersonExtraFactory.create(base__id="4322",
                                                      base__name="Helen Hayes")
        edinburgh_candidate = PersonExtraFactory.create(
            base__id="818", base__name="Sheila Gilmore")
        edinburgh_winner = PersonExtraFactory.create(
            base__id="5795", base__name="Tommy Sheppard")
        edinburgh_may_stand = PersonExtraFactory.create(
            base__id="5163", base__name="Peter McColl")
        MembershipFactory.create(
            person=person_extra.base,
            post=self.dulwich_post_extra.base,
            on_behalf_of=self.labour_party_extra.base,
            post_election=self.dulwich_post_extra_pee,
        )
        MembershipFactory.create(
            person=person_extra.base,
            organization=self.labour_party_extra.base,
            post_election=self.edinburgh_east_post_extra_pee,
        )

        MembershipFactory.create(
            person=dulwich_not_stand.base,
            post=self.dulwich_post_extra.base,
            on_behalf_of=self.labour_party_extra.base,
            post_election=self.dulwich_post_extra_pee_earlier,
        )
        dulwich_not_stand.not_standing.add(self.election)

        MembershipFactory.create(
            person=edinburgh_winner.base,
            post=self.edinburgh_east_post_extra.base,
            on_behalf_of=self.labour_party_extra.base,
            elected=True,
            post_election=self.edinburgh_east_post_extra_pee,
        )

        MembershipFactory.create(
            person=edinburgh_candidate.base,
            post=self.edinburgh_east_post_extra.base,
            on_behalf_of=self.labour_party_extra.base,
            post_election=self.edinburgh_east_post_extra_pee,
        )

        MembershipFactory.create(
            person=edinburgh_may_stand.base,
            post=self.edinburgh_east_post_extra.base,
            on_behalf_of=self.labour_party_extra.base,
            post_election=self.edinburgh_east_post_extra_pee_earlier,
        )

        self.storage = DefaultStorage()

    def test_api_basic_response(self):
        response = self.app.get("/api/v0.9/")
        self.assertEqual(response.status_code, 200)
        json = response.json

        self.assertEqual(json["persons"],
                         "http://localhost:80/api/v0.9/persons/")
        self.assertEqual(json["organizations"],
                         "http://localhost:80/api/v0.9/organizations/")
        self.assertEqual(json["elections"],
                         "http://localhost:80/api/v0.9/elections/")
        self.assertEqual(json["posts"], "http://localhost:80/api/v0.9/posts/")

        persons_resp = self.app.get("/api/v0.9/persons/")
        self.assertEqual(persons_resp.status_code, 200)

        organizations_resp = self.app.get("/api/v0.9/organizations/")
        self.assertEqual(organizations_resp.status_code, 200)

        elections_resp = self.app.get("/api/v0.9/elections/")
        self.assertEqual(elections_resp.status_code, 200)

        posts_resp = self.app.get("/api/v0.9/posts/")
        self.assertEqual(posts_resp.status_code, 200)

    def test_api_errors(self):
        response = self.app.get("/api/", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.get("/api/v0.8", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.get("/api/v0.9/person/", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.get("/api/v0.9/persons/4000/", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.post("/api/v0.9/persons/", {}, expect_errors=True)
        self.assertEqual(response.status_code, 403)

    def test_api_persons(self):
        persons_resp = self.app.get("/api/v0.9/persons/")

        persons = persons_resp.json

        self.assertEqual(persons["count"], len(persons["results"]))
        self.assertEqual(persons["count"], 5)

    def test_api_person(self):
        person_resp = self.app.get("/api/v0.9/persons/2009/")

        self.assertEqual(person_resp.status_code, 200)

        person = person_resp.json
        self.assertEqual(person["id"], 2009)
        self.assertEqual(person["name"], "Tessa Jowell")

        memberships = sorted(person["memberships"], key=lambda m: m["role"])

        self.assertEqual(len(memberships), 2)
        self.assertEqual(memberships[1]["role"], "Candidate")

        self.assertEqual(len(person["versions"]), 0)

    def test_api_organizations(self):
        organizations_resp = self.app.get("/api/v0.9/organizations/")

        organizations = organizations_resp.json

        self.assertEqual(organizations["count"], len(organizations["results"]))
        self.assertEqual(organizations["count"], 7)

    def test_api_organization(self):
        organizations_resp = self.app.get("/api/v0.9/organizations/")
        organizations = organizations_resp.json

        organization_url = None
        for organization in organizations["results"]:
            if organization["id"] == "party:53":
                organization_url = organization["url"]
                break

        organization_resp = self.app.get(organization_url)
        self.assertEqual(organization_resp.status_code, 200)

        organization = organization_resp.json
        self.assertEqual(organization["id"], "party:53")
        self.assertEqual(organization["name"], "Labour Party")

    def test_api_elections(self):
        elections_resp = self.app.get("/api/v0.9/elections/")

        elections = elections_resp.json

        self.assertEqual(elections["count"], len(elections["results"]))
        self.assertEqual(elections["count"], 3)

    def test_api_election(self):
        elections_resp = self.app.get("/api/v0.9/elections/")
        elections = elections_resp.json

        election_url = None
        for election in elections["results"]:
            if election["id"] == "2015":
                election_url = election["url"]
                break

        election_resp = self.app.get(election_url)
        self.assertEqual(election_resp.status_code, 200)

        election = election_resp.json
        self.assertEqual(election["id"], "2015")
        self.assertEqual(election["name"], "2015 General Election")

    def test_api_posts(self):
        posts_resp = self.app.get("/api/v0.9/posts/")

        posts = posts_resp.json

        self.assertEqual(posts["count"], len(posts["results"]))
        self.assertEqual(posts["count"], 5)

    def test_api_post(self):
        posts_resp = self.app.get("/api/v0.9/posts/")
        posts = posts_resp.json

        post_url = None
        for post in posts["results"]:
            if post["id"] == "65808":
                post_url = post["url"]
                break

        self.assertTrue(post_url)
        post_resp = self.app.get(post_url)
        self.assertEqual(post_resp.status_code, 200)

        post = post_resp.json

        self.assertEqual(post["id"], "65808")
        self.assertEqual(post["label"],
                         "Member of Parliament for Dulwich and West Norwood")

    def test_api_person_redirects(self):
        PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
        PersonRedirect.objects.create(old_person_id="5678", new_person_id="12")
        person_redirects_resp = self.app.get("/api/v0.9/person_redirects/")
        person_redirects = person_redirects_resp.json
        self.assertEqual(person_redirects["results"][0]["old_person_id"], 1234)
        self.assertEqual(person_redirects["results"][0]["new_person_id"], 42)
        self.assertEqual(person_redirects["results"][1]["old_person_id"], 5678)
        self.assertEqual(person_redirects["results"][1]["new_person_id"], 12)

    def test_api_person_redirect(self):
        PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
        url = "/api/v0.9/person_redirects/1234/"
        person_redirect_resp = self.app.get(url)
        person_redirect = person_redirect_resp.json
        self.assertEqual(person_redirect["old_person_id"], 1234)
        self.assertEqual(person_redirect["new_person_id"], 42)

    def test_api_version_info(self):
        version_resp = self.app.get("/version.json")
        self.assertEqual(version_resp.status_code, 200)

        info = version_resp.json
        self.assertEqual(info["users_who_have_edited"], 0)
        self.assertEqual(info["interesting_user_actions"], 0)

        LoggedAction.objects.create(action_type="set-candidate-not-elected")

        LoggedAction.objects.create(action_type="edit-candidate")

        version_resp = self.app.get("/version.json")
        info = version_resp.json
        self.assertEqual(info["interesting_user_actions"], 1)

    def test_api_cors_headers(self):
        resp = self.app.get("/api/v0.9/",
                            headers={"Origin": b"http://example.com"})
        self.assertTrue("Access-Control-Allow-Origin" in resp.headers)
        self.assertEqual(resp.headers["Access-Control-Allow-Origin"], "*")

        resp = self.app.get("/", headers={"Origin": b"http://example.com"})
        self.assertFalse("Access-Control-Allow-Origin" in resp.headers)

    def test_api_jsonp_response(self):
        response = self.app.get("/api/v0.9/?format=jsonp&callback=test")
        self.assertEqual(response.status_code, 200)
        self.assertTrue(response.text.startswith("test("))

    @patch(
        "candidates.management.commands.candidates_cache_api_to_directory.datetime"
    )
    def test_persons_api_to_directory(self, mock_datetime):
        # current
        # timestamped
        # timestamped
        mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
        target_directory = settings.MEDIA_ROOT
        call_command(
            "candidates_cache_api_to_directory",
            page_size="3",
            url_prefix="https://example.com/media/api-cache-for-wcivf/",
        )
        expected_leafname = "2017-05-14T12:33:05"
        expected_timestamped_directory = join(settings.MEDIA_ROOT,
                                              "cached-api", expected_leafname)
        expected_path = join("cached-api", "latest")
        self.assertTrue(self.storage.exists(expected_timestamped_directory))
        self.assertTrue(self.storage.exists(expected_path))
        # Check that the files in that directory are as expected:
        entries = self.storage.listdir(expected_timestamped_directory)[1]
        persons_1_leafname = "persons-000001.json"
        persons_2_leafname = "persons-000002.json"
        posts_1_leafname = "posts-000001.json"
        posts_2_leafname = "posts-000002.json"
        self.assertEqual(
            set(entries),
            {
                persons_1_leafname,
                persons_2_leafname,
                posts_1_leafname,
                posts_2_leafname,
            },
        )
        # Get the data from those pages:
        with self.storage.open(
                join(expected_timestamped_directory, persons_1_leafname)) as f:
            persons_1_data = json.loads(f.read().decode('utf8'))
        with self.storage.open(
                join(expected_timestamped_directory, persons_2_leafname)) as f:
            persons_2_data = json.loads(f.read().decode('utf8'))
        with self.storage.open(
                join(expected_timestamped_directory, posts_1_leafname)) as f:
            posts_1_data = json.loads(f.read().decode('utf8'))
        with self.storage.open(
                join(expected_timestamped_directory, posts_2_leafname)) as f:
            posts_2_data = json.loads(f.read().decode('utf8'))
        # Check the previous and next links are as we expect:
        self.assertEqual(
            persons_1_data["next"],
            "https://example.com/media/api-cache-for-wcivf/{}/{}".format(
                expected_leafname, persons_2_leafname),
        )
        self.assertEqual(persons_1_data["previous"], None)
        self.assertEqual(persons_2_data["next"], None)
        self.assertEqual(
            persons_2_data["previous"],
            "https://example.com/media/api-cache-for-wcivf/{}/{}".format(
                expected_leafname, persons_1_leafname),
        )
        self.assertEqual(
            posts_1_data["next"],
            "https://example.com/media/api-cache-for-wcivf/{}/{}".format(
                expected_leafname, posts_2_leafname),
        )
        self.assertEqual(posts_1_data["previous"], None)
        self.assertEqual(posts_2_data["next"], None)
        self.assertEqual(
            posts_2_data["previous"],
            "https://example.com/media/api-cache-for-wcivf/{}/{}".format(
                expected_leafname, posts_1_leafname),
        )
        # Check that the URL of the first person is as expected,
        # as well as it being the right person:
        first_person = persons_1_data["results"][0]
        self.assertEqual(first_person["id"], 818)
        self.assertEqual(first_person["name"], "Sheila Gilmore")
        self.assertEqual(
            first_person["url"],
            "https://candidates.democracyclub.org.uk/api/v0.9/persons/818/?format=json",
        )
        # Similarly, check that the URL of the first post is as expected:
        first_post = posts_1_data["results"][0]
        self.assertEqual(first_post["id"], self.edinburgh_east_post_extra.slug)
        self.assertEqual(first_post["label"],
                         "Member of Parliament for Edinburgh East")
        self.assertEqual(
            first_post["url"],
            "https://candidates.democracyclub.org.uk/api/v0.9/posts/14419/?format=json",
        )

    def _setup_cached_api_directory(self, dir_list):
        """
        Saves a tmp file in settings.MEDIA_ROOT, called `.keep` in each
        directory in dir_list.

        """
        for d in dir_list:
            self.storage.save(join("cached-api", d, ".keep"), ContentFile("."))

    @patch(
        "candidates.management.commands.candidates_cache_api_to_directory.datetime"
    )
    def test_cache_api_to_directory_prune(self, mock_datetime):
        # Need to make sure datetime.strptime still works:
        mock_datetime.strptime.side_effect = datetime.strptime
        mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)

        expected_to_prune = [
            "2017-05-12T08:00:00",
            "2017-05-12T10:00:00",
            "2017-05-12T12:00:00",
        ]
        expected_to_keep = [
            "2017-05-14T08:00:00",
            "2017-05-14T10:00:00",
            "2017-05-14T12:00:00",
            "2017-05-14T12:33:05",
        ]

        self._setup_cached_api_directory(expected_to_keep + expected_to_prune)

        self.assertTrue(
            self.storage.exists(
                join("cached-api", "2017-05-12T08:00:00", ".keep")))

        call_command(
            "candidates_cache_api_to_directory",
            page_size="3",
            url_prefix="https://example.com/media/api-cache-for-wcivf/",
            prune=True,
        )

        for dir_name in expected_to_keep:
            self.assertTrue(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

        for dir_name in expected_to_prune:
            self.assertFalse(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

    @patch(
        "candidates.management.commands.candidates_cache_api_to_directory.datetime"
    )
    def test_cache_api_to_directory_prune_four_old(self, mock_datetime):
        # Need to make sure datetime.strptime still works:
        mock_datetime.strptime.side_effect = datetime.strptime
        mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)

        expected_to_prune = ["2017-05-12T06:00:00"]
        expected_to_keep = [
            "2017-05-12T08:00:00",
            "2017-05-12T10:00:00",
            "2017-05-12T12:00:00",
            "2017-05-14T12:33:05",
        ]

        self._setup_cached_api_directory(expected_to_keep + expected_to_prune)

        self.assertTrue(
            self.storage.exists(
                join("cached-api", "2017-05-12T06:00:00", ".keep")))

        self.assertTrue(
            self.storage.exists(
                join("cached-api", "2017-05-12T08:00:00", ".keep")))

        call_command(
            "candidates_cache_api_to_directory",
            page_size="3",
            url_prefix="https://example.com/media/api-cache-for-wcivf/",
            prune=True,
        )
        # Even though all of those directories are more than 36
        # hours old, they should all be kept because they're the
        # most recent 4:
        for dir_name in expected_to_keep:
            self.assertTrue(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

        for dir_name in expected_to_prune:
            self.assertFalse(
                self.storage.exists(join("cached-api", dir_name, ".keep")))
Пример #11
0
class CSVTests(TmpMediaRootMixin, TestUserMixin, UK2015ExamplesMixin, TestCase):
    def setUp(self):
        super().setUp()
        self.storage = DefaultStorage()
        # The second person's name (and party name) have diacritics in
        # them to test handling of Unicode when outputting to CSV.
        self.gb_person = people.tests.factories.PersonFactory.create(
            id=2009,
            name="Tessa Jowell",
            honorific_suffix="DBE",
            honorific_prefix="Ms",
            gender="female",
        )
        PersonImage.objects.create_from_file(
            EXAMPLE_IMAGE_FILENAME,
            "images/jowell-pilot.jpg",
            defaults={
                "person": self.gb_person,
                "is_primary": True,
                "source": "Taken from Wikipedia",
                "copyright": "example-license",
                "uploading_user": self.user,
                "user_notes": "A photo of Tessa Jowell",
            },
        )

        self.ni_person = people.tests.factories.PersonFactory.create(
            id=1953, name="Daithí McKay", gender="male"
        )
        north_antrim_post = factories.PostFactory.create(
            elections=(self.election, self.earlier_election),
            organization=self.commons,
            slug="66135",
            label="Member of Parliament for North Antrim",
            party_set=self.ni_parties,
        )
        factories.MembershipFactory.create(
            person=self.ni_person,
            post=north_antrim_post,
            party=self.sinn_fein,
            post_election=self.election.postextraelection_set.get(
                post=north_antrim_post
            ),
        )
        factories.MembershipFactory.create(
            person=self.ni_person,
            post=north_antrim_post,
            party=self.sinn_fein,
            post_election=self.earlier_election.postextraelection_set.get(
                post=north_antrim_post
            ),
        )
        factories.MembershipFactory.create(
            person=self.gb_person,
            post=self.camberwell_post,
            party=self.labour_party,
            post_election=self.camberwell_post_pee,
        )
        factories.MembershipFactory.create(
            person=self.gb_person,
            post=self.dulwich_post,
            party=self.labour_party,
            post_election=self.dulwich_post_pee_earlier,
        )

        self.gb_person.tmp_person_identifiers.create(
            internal_identifier="10326", value_type="theyworkforyou"
        )
        self.gb_person.tmp_person_identifiers.create(
            value_type="email", value="*****@*****.**"
        )

    def test_as_single_dict(self):
        membership = (
            Membership.objects.for_csv()
            .filter(person=self.gb_person.id)
            .first()
        )
        # After the select_related and prefetch_related calls
        # PersonExtra there should only be one more query - that to
        # find the complex fields mapping:
        with self.assertNumQueries(0):
            membership_dict = membership.dict_for_csv()

        self.assertEqual(
            sorted(list(membership_dict.keys())),
            sorted(settings.CSV_ROW_FIELDS),
        )
        self.assertEqual(len(membership_dict.keys()), 39)
        self.assertEqual(membership_dict["id"], 2009)

        self.assertEqual(
            membership_dict["parlparse_id"], "uk.org.publicwhip/person/10326"
        )
        self.assertEqual(
            membership_dict["theyworkforyou_url"],
            "http://www.theyworkforyou.com/mp/10326",
        )

    def test_as_dict_2010(self):
        with self.assertNumQueries(5):
            memberships_dicts, elected = memberships_dicts_for_csv(
                self.earlier_election.slug
            )
        self.assertEqual(len(memberships_dicts[self.earlier_election.slug]), 2)
        membership_dict = memberships_dicts["parl.2010-05-06"][1]
        self.assertEqual(len(membership_dict.keys()), 39)
        self.assertEqual(membership_dict["id"], 2009)

    def test_csv_output(self):
        tessa_image_url = self.gb_person.primary_image.url
        d = {
            "election_date": date_in_near_future,
            "earlier_election_date": date_in_near_future
            - timedelta(days=FOUR_YEARS_IN_DAYS),
        }
        PersonRedirect.objects.create(old_person_id=12, new_person_id=1953)
        PersonRedirect.objects.create(old_person_id=56, new_person_id=1953)
        self.maxDiff = None
        example_output = (
            "id,name,honorific_prefix,honorific_suffix,gender,birth_date,election,party_id,party_name,post_id,post_label,mapit_url,elected,email,twitter_username,facebook_page_url,party_ppc_page_url,facebook_personal_url,homepage_url,wikipedia_url,linkedin_url,image_url,proxy_image_url_template,image_copyright,image_uploading_user,image_uploading_user_notes,twitter_user_id,election_date,election_current,party_lists_in_use,party_list_position,old_person_ids,gss_code,parlparse_id,theyworkforyou_url,party_ec_id,favourite_biscuits,cancelled_poll,wikidata_url\r\n"
            + "1953,Daith\xed McKay,,,male,,parl.2010-05-06,party:39,Sinn F\xe9in,66135,North Antrim,,,,,,,,,,,,,,,,,{earlier_election_date},False,False,,12;56,,,,PP39,,False,\r\n".format(
                **d
            )
            + "2009,Tessa Jowell,Ms,DBE,female,,parl.2010-05-06,party:53,Labour Party,65808,Dulwich and West Norwood,,,[email protected],,,,,,,,{image_url},,example-license,john,A photo of Tessa Jowell,,{earlier_election_date},False,False,,,,uk.org.publicwhip/person/10326,http://www.theyworkforyou.com/mp/10326,PP53,,False,\r\n".format(
                image_url=tessa_image_url, **d
            )
            + "1953,Daith\xed McKay,,,male,,parl.2015-05-07,party:39,Sinn F\xe9in,66135,North Antrim,,,,,,,,,,,,,,,,,{election_date},True,False,,12;56,,,,PP39,,False,\r\n".format(
                **d
            )
            + "2009,Tessa Jowell,Ms,DBE,female,,parl.2015-05-07,party:53,Labour Party,65913,Camberwell and Peckham,,,[email protected],,,,,,,,{image_url},,example-license,john,A photo of Tessa Jowell,,{election_date},True,False,,,,uk.org.publicwhip/person/10326,http://www.theyworkforyou.com/mp/10326,PP53,,False,\r\n".format(
                image_url=tessa_image_url, **d
            )
        )

        with self.assertNumQueries(5):
            memberships_dicts, elected = memberships_dicts_for_csv()
        all_members = []
        for slug, members in memberships_dicts.items():
            all_members += members
        self.assertEqual(list_to_csv(all_members), example_output)

    def test_create_csv_management_command(self):
        # An empty media directory
        self.assertEqual(
            self.storage.listdir(self.storage.base_location)[1], []
        )

        # We expect a CSV file per election, and one for all elections
        call_command("candidates_create_csv")
        self.assertEqual(
            set(sorted(self.storage.listdir(".")[1])),
            set(
                [
                    "candidates-parl.2010-05-06.csv",
                    "candidates-parl.2015-05-07.csv",
                    "candidates-all.csv",
                    "candidates-elected-all.csv",
                    "candidates-local.maidstone.2016-05-05.csv",
                    "candidates-2015-05-07.csv",
                    "candidates-2010-05-06.csv",
                ]
            ),
        )

    def test_create_csv_management_command_single_election(self):
        # An empty media directory
        self.assertEqual(
            self.storage.listdir(self.storage.base_location), (["images"], [])
        )

        # We expect a single CSV file
        call_command("candidates_create_csv", "--election", "parl.2015-05-07")
        self.assertSetEqual(
            set(self.storage.listdir(".")[1]),
            set(
                [
                    "candidates-parl.2010-05-06.csv",
                    "candidates-parl.2015-05-07.csv",
                    "candidates-local.maidstone.2016-05-05.csv",
                    "candidates-2015-05-07.csv",
                ]
            ),
        )

    def test_create_csv_management_command_single_election_doesnt_exist(self):
        # An empty media directory, apart from the images dir
        self.assertEqual(
            self.storage.listdir(self.storage.base_location), (["images"], [])
        )

        with self.assertRaises(CommandError):
            call_command("candidates_create_csv", "--election", "foo")

    def test_create_csv_no_candidates_for_election(self):
        # An empty media directory, apart from the images dir
        self.assertEqual(
            self.storage.listdir(self.storage.base_location), (["images"], [])
        )

        factories.ElectionFactory.create(
            slug="2018",
            name="2018 General Election",
            for_post_role="Member of Parliament",
            organization=self.commons,
        )

        call_command("candidates_create_csv")
        self.assertSetEqual(
            set(self.storage.listdir(".")[1]),
            set(
                [
                    "candidates-parl.2010-05-06.csv",
                    "candidates-parl.2015-05-07.csv",
                    "candidates-2018.csv",
                    "candidates-all.csv",
                    "candidates-elected-all.csv",
                    "candidates-local.maidstone.2016-05-05.csv",
                    "candidates-2015-05-07.csv",
                    "candidates-2010-05-06.csv",
                ]
            ),
        )
        empty_file = self.storage.open("candidates-2018.csv").read()
        self.assertEqual(len(empty_file.splitlines()), 1)
        self.assertEqual(
            empty_file.splitlines()[0].decode(),
            (",".join(settings.CSV_ROW_FIELDS)),
        )

        non_empty_file = self.storage.open(
            "candidates-parl.2015-05-07.csv"
        ).read()
        self.assertEqual(len(non_empty_file.splitlines()), 3)
Пример #12
0
class TestAPI(TestUserMixin, TmpMediaRootMixin, UK2015ExamplesMixin, WebTest):
    def setUp(self):
        super().setUp()

        person = PersonFactory.create(id=2009, name="Tessa Jowell")
        PersonImage.objects.update_or_create_from_file(
            EXAMPLE_IMAGE_FILENAME,
            "images/imported.jpg",
            person,
            defaults={
                "md5sum": "md5sum",
                "copyright": "example-license",
                "uploading_user": self.user,
                "user_notes": "Here's an image...",
                "is_primary": True,
                "source": "Found on the candidate's Flickr feed",
            },
        )
        dulwich_not_stand = PersonFactory.create(id=4322, name="Helen Hayes")
        edinburgh_candidate = PersonFactory.create(id="818",
                                                   name="Sheila Gilmore")
        edinburgh_winner = PersonFactory.create(id="5795",
                                                name="Tommy Sheppard")
        edinburgh_may_stand = PersonFactory.create(id="5163",
                                                   name="Peter McColl")
        MembershipFactory.create(
            person=person,
            post=self.dulwich_post,
            party=self.labour_party,
            ballot=self.dulwich_post_ballot,
        )
        MembershipFactory.create(person=person,
                                 ballot=self.edinburgh_east_post_ballot)

        MembershipFactory.create(
            person=dulwich_not_stand,
            post=self.dulwich_post,
            party=self.labour_party,
            ballot=self.dulwich_post_ballot_earlier,
        )
        dulwich_not_stand.not_standing.add(self.election)

        MembershipFactory.create(
            person=edinburgh_winner,
            post=self.edinburgh_east_post,
            party=self.labour_party,
            elected=True,
            ballot=self.edinburgh_east_post_ballot,
        )

        MembershipFactory.create(
            person=edinburgh_candidate,
            post=self.edinburgh_east_post,
            party=self.labour_party,
            ballot=self.edinburgh_east_post_ballot,
        )

        MembershipFactory.create(
            person=edinburgh_may_stand,
            post=self.edinburgh_east_post,
            party=self.labour_party,
            ballot=self.edinburgh_east_post_ballot_earlier,
        )

        self.storage = DefaultStorage()

    def test_api_basic_response(self):
        response = self.app.get("/api/v0.9/")
        self.assertEqual(response.status_code, 200)
        json = response.json

        self.assertEqual(json["persons"],
                         "http://testserver/api/v0.9/persons/")
        self.assertEqual(json["organizations"],
                         "http://testserver/api/v0.9/organizations/")
        self.assertEqual(json["elections"],
                         "http://testserver/api/v0.9/elections/")
        self.assertEqual(json["posts"], "http://testserver/api/v0.9/posts/")

        persons_resp = self.app.get("/api/v0.9/persons/")
        self.assertEqual(persons_resp.status_code, 200)

        organizations_resp = self.app.get("/api/v0.9/organizations/")
        self.assertEqual(organizations_resp.status_code, 200)

        elections_resp = self.app.get("/api/v0.9/elections/")
        self.assertEqual(elections_resp.status_code, 200)

        posts_resp = self.app.get("/api/v0.9/posts/")
        self.assertEqual(posts_resp.status_code, 200)

    def test_api_home(self):
        response = self.app.get("/api/")
        self.assertEqual(response.status_code, 302)

        response = self.app.get("/api/docs/")
        self.assertEqual(response.status_code, 200)

        response = self.app.get("/api/v0.8", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.get("/api/v0.9/person/", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.get("/api/v0.9/persons/4000/", expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.app.post("/api/v0.9/persons/",
                                 params={},
                                 expect_errors=True)
        self.assertEqual(response.status_code, 403)

    def test_api_persons(self):
        persons_resp = self.app.get("/api/v0.9/persons/")

        persons = persons_resp.json

        self.assertEqual(persons["count"], len(persons["results"]))
        self.assertEqual(persons["count"], 5)

    def test_api_person(self):
        person_resp = self.app.get("/api/v0.9/persons/2009/")

        self.assertEqual(person_resp.status_code, 200)

        person = person_resp.json
        self.assertEqual(person["id"], 2009)
        self.assertEqual(person["name"], "Tessa Jowell")

        memberships = sorted(person["memberships"], key=lambda m: m["role"])

        self.assertEqual(len(memberships), 2)
        self.assertEqual(memberships[1]["role"], "Candidate")

        self.assertEqual(len(person["versions"]), 0)

    def _make_legacy_parties(self):
        """
        It used to be that political parties were stored on the "Organization"
        model. for maintaining v0.9 API compatibility we've not deleted them
        from that model (yet), so let's make the test data support this legacy
        case
        """

        from candidates.tests.factories import OrganizationFactory
        from candidates.tests.uk_examples import EXAMPLE_PARTIES

        for party in EXAMPLE_PARTIES:
            p = OrganizationFactory(slug=party["legacy_slug"],
                                    name=party["name"])

    def test_api_legacy_organizations_with_parties(self):
        self._make_legacy_parties()
        organizations_resp = self.app.get("/api/v0.9/organizations/")

        organizations = organizations_resp.json

        self.assertEqual(organizations["count"], len(organizations["results"]))
        self.assertEqual(organizations["count"], 7)

    def test_api_legacy_organization_with_parties(self):
        self._make_legacy_parties()
        organizations_resp = self.app.get("/api/v0.9/organizations/")
        organizations = organizations_resp.json

        organization_url = None
        for organization in organizations["results"]:
            if organization["id"] == "party:53":
                organization_url = organization["url"]
                break

        organization_resp = self.app.get(organization_url)
        self.assertEqual(organization_resp.status_code, 200)

        organization = organization_resp.json
        self.assertEqual(organization["id"], "party:53")
        self.assertEqual(organization["name"], "Labour Party")

    def test_api_elections(self):
        elections_resp = self.app.get("/api/v0.9/elections/")

        elections = elections_resp.json

        self.assertEqual(elections["count"], len(elections["results"]))
        self.assertEqual(elections["count"], 3)

    def test_api_elections_without_orgs(self):
        # Regression test that we can serve elections without an organzation
        self.election.organization = None
        self.election.save()
        elections_resp = self.app.get("/api/v0.9/elections/",
                                      expect_errors=True)
        self.assertEqual(elections_resp.status_code, 200)

    def test_api_election(self):
        elections_resp = self.app.get("/api/v0.9/elections/")
        elections = elections_resp.json

        election_url = None
        for election in elections["results"]:
            if election["id"] == "parl.2015-05-07":
                election_url = election["url"]
                break

        election_resp = self.app.get(election_url)
        self.assertEqual(election_resp.status_code, 200)

        election = election_resp.json
        self.assertEqual(election["id"], "parl.2015-05-07")
        self.assertEqual(election["name"], "2015 General Election")

    def test_api_posts(self):
        posts_resp = self.app.get("/api/v0.9/posts/")

        posts = posts_resp.json

        self.assertEqual(posts["count"], len(posts["results"]))
        self.assertEqual(posts["count"], 5)

    def test_api_post(self):
        posts_resp = self.app.get("/api/v0.9/posts/")
        posts = posts_resp.json

        post_url = None
        for post in posts["results"]:
            if post["id"] == "65808":
                post_url = post["url"]
                break

        self.assertTrue(post_url)
        post_resp = self.app.get(post_url)
        self.assertEqual(post_resp.status_code, 200)

        post = post_resp.json

        self.assertEqual(post["id"], "65808")
        self.assertEqual(post["label"],
                         "Member of Parliament for Dulwich and West Norwood")

    def test_api_person_redirects(self):
        PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
        PersonRedirect.objects.create(old_person_id="5678", new_person_id="12")
        person_redirects_resp = self.app.get("/api/v0.9/person_redirects/")
        person_redirects = person_redirects_resp.json
        self.assertEqual(person_redirects["results"][0]["old_person_id"], 1234)
        self.assertEqual(person_redirects["results"][0]["new_person_id"], 42)
        self.assertEqual(person_redirects["results"][1]["old_person_id"], 5678)
        self.assertEqual(person_redirects["results"][1]["new_person_id"], 12)

    def test_api_person_redirect(self):
        PersonRedirect.objects.create(old_person_id="1234", new_person_id="42")
        url = "/api/v0.9/person_redirects/1234/"
        person_redirect_resp = self.app.get(url)
        person_redirect = person_redirect_resp.json
        self.assertEqual(person_redirect["old_person_id"], 1234)
        self.assertEqual(person_redirect["new_person_id"], 42)

    def test_api_version_info(self):
        version_resp = self.app.get("/version.json")
        self.assertEqual(version_resp.status_code, 200)

        info = version_resp.json
        self.assertEqual(info["users_who_have_edited"], 0)
        self.assertEqual(info["interesting_user_actions"], 0)

        LoggedAction.objects.create(action_type="set-candidate-not-elected")

        LoggedAction.objects.create(action_type="edit-candidate")

        version_resp = self.app.get("/version.json")
        info = version_resp.json
        self.assertEqual(info["interesting_user_actions"], 1)

    def test_api_cors_headers(self):
        resp = self.app.get("/api/v0.9/",
                            headers={"Origin": b"http://example.com"})
        self.assertTrue("Access-Control-Allow-Origin" in resp.headers)
        self.assertEqual(resp.headers["Access-Control-Allow-Origin"], "*")

        resp = self.app.get("/", headers={"Origin": b"http://example.com"})
        self.assertFalse("Access-Control-Allow-Origin" in resp.headers)

    def test_api_jsonp_response(self):
        response = self.app.get("/api/v0.9/?format=jsonp&callback=test")
        self.assertEqual(response.status_code, 200)
        self.assertTrue(response.text.startswith("test("))

    @patch(
        "candidates.management.commands.candidates_cache_api_to_directory.datetime"
    )
    def test_persons_api_to_directory(self, mock_datetime):
        # current
        # timestamped
        # timestamped
        mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)
        target_directory = settings.MEDIA_ROOT
        call_command(
            "candidates_cache_api_to_directory",
            page_size="3",
            url_prefix="https://example.com/media/api-cache-for-wcivf",
        )
        expected_leafname = "2017-05-14T12:33:05"
        expected_timestamped_directory = join(settings.MEDIA_ROOT,
                                              "cached-api", expected_leafname)
        expected_path = join("cached-api", "latest")
        self.assertTrue(self.storage.exists(expected_timestamped_directory))
        self.assertTrue(self.storage.exists(expected_path))
        # Check that the files in that directory are as expected:
        entries = self.storage.listdir(expected_timestamped_directory)[1]
        people_1_leafname = "people-000001.json"
        people_2_leafname = "people-000002.json"

        ballot_1_leafname = "ballots-000001.json"
        ballot_2_leafname = "ballots-000002.json"
        ballot_3_leafname = "ballots-000003.json"

        self.assertEqual(
            set(entries),
            {
                people_1_leafname,
                people_2_leafname,
                ballot_1_leafname,
                ballot_2_leafname,
                ballot_3_leafname,
            },
        )
        # Get the data from those pages:
        with self.storage.open(
                join(expected_timestamped_directory, people_1_leafname)) as f:
            persons_1_data = json.loads(f.read().decode("utf8"))
        with self.storage.open(
                join(expected_timestamped_directory, people_2_leafname)) as f:
            persons_2_data = json.loads(f.read().decode("utf8"))

        # Check the previous and next links are as we expect:
        self.assertEqual(
            persons_1_data["next"],
            "https://example.com/media/api-cache-for-wcivf/{}/{}".format(
                expected_leafname, people_2_leafname),
        )
        self.assertEqual(persons_1_data["previous"], None)
        self.assertEqual(persons_2_data["next"], None)
        self.assertEqual(
            persons_2_data["previous"],
            "https://example.com/media/api-cache-for-wcivf/{}/{}".format(
                expected_leafname, people_1_leafname),
        )
        # Check that the URL of the first person is as expected,
        # as well as it being the right person:
        first_person = persons_1_data["results"][0]
        self.assertEqual(first_person["id"], 818)
        self.assertEqual(first_person["name"], "Sheila Gilmore")
        self.assertEqual(
            first_person["url"],
            "https://candidates.democracyclub.org.uk/api/next/people/818/?format=json",
        )

    def _setup_cached_api_directory(self, dir_list):
        """
        Saves a tmp file in settings.MEDIA_ROOT, called `.keep` in each
        directory in dir_list.

        """
        for d in dir_list:
            self.storage.save(join("cached-api", d, ".keep"), ContentFile("."))

    @patch(
        "candidates.management.commands.candidates_cache_api_to_directory.datetime"
    )
    def test_cache_api_to_directory_prune(self, mock_datetime):
        # Need to make sure datetime.strptime still works:
        mock_datetime.strptime.side_effect = datetime.strptime
        mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)

        expected_to_prune = [
            "2017-05-12T08:00:00",
            "2017-05-12T10:00:00",
            "2017-05-12T12:00:00",
        ]
        expected_to_keep = [
            "2017-05-14T08:00:00",
            "2017-05-14T10:00:00",
            "2017-05-14T12:00:00",
            "2017-05-14T12:33:05",
        ]

        self._setup_cached_api_directory(expected_to_keep + expected_to_prune)

        self.assertTrue(
            self.storage.exists(
                join("cached-api", "2017-05-12T08:00:00", ".keep")))

        call_command(
            "candidates_cache_api_to_directory",
            page_size="3",
            url_prefix="https://example.com/media/api-cache-for-wcivf/",
            prune=True,
        )

        for dir_name in expected_to_keep:
            self.assertTrue(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

        for dir_name in expected_to_prune:
            self.assertFalse(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

    @patch(
        "candidates.management.commands.candidates_cache_api_to_directory.datetime"
    )
    def test_cache_api_to_directory_prune_four_old(self, mock_datetime):
        # Need to make sure datetime.strptime still works:
        mock_datetime.strptime.side_effect = datetime.strptime
        mock_datetime.now.return_value = datetime(2017, 5, 14, 12, 33, 5, 0)

        expected_to_prune = ["2017-05-12T06:00:00"]
        expected_to_keep = [
            "2017-05-12T08:00:00",
            "2017-05-12T10:00:00",
            "2017-05-12T12:00:00",
            "2017-05-14T12:33:05",
        ]

        self._setup_cached_api_directory(expected_to_keep + expected_to_prune)

        self.assertTrue(
            self.storage.exists(
                join("cached-api", "2017-05-12T06:00:00", ".keep")))

        self.assertTrue(
            self.storage.exists(
                join("cached-api", "2017-05-12T08:00:00", ".keep")))

        call_command(
            "candidates_cache_api_to_directory",
            page_size="3",
            url_prefix="https://example.com/media/api-cache-for-wcivf/",
            prune=True,
        )
        # Even though all of those directories are more than 36
        # hours old, they should all be kept because they're the
        # most recent 4:
        for dir_name in expected_to_keep:
            self.assertTrue(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

        for dir_name in expected_to_prune:
            self.assertFalse(
                self.storage.exists(join("cached-api", dir_name, ".keep")))

    def test_legacy_redirects(self):
        req = self.app.get("/api/v0.9/elections/2010/")
        self.assertEqual(req.status_code, 301)
        self.assertEqual(req.location, "/api/v0.9/elections/parl.2010-05-06/")
        req = self.app.get("/api/v0.9/elections/2010.json")
        self.assertEqual(req.status_code, 301)
        self.assertEqual(req.location,
                         "/api/v0.9/elections/parl.2010-05-06.json")

    def test_legacy_contact_details(self):
        person = PersonFactory()
        PersonIdentifier.objects.create(value_type="twitter_username",
                                        value="Froglet4MP",
                                        person=person)

        req = self.app.get("/api/v0.9/persons/{}/".format(person.pk))
        person_json = req.json
        self.assertTrue("contact_details" in person_json)
        self.assertEqual(person_json["contact_details"][0]["contact_type"],
                         "twitter")
        self.assertEqual(person_json["contact_details"][0]["value"],
                         "Froglet4MP")