def test_get_next_url_should_call_apply_all_rules_on_url(self):
     def_rule = Mock(spec=DefaultRule)
     def_rule.apply_rule_to.return_value = "http://localhost/2020"
     generation_rule = Mock(spec=LimitOffset)
     generation_rule.apply_rule_to.return_value = "http://localhost?limit=5&offset=10"
     url_extractor = UrlExtractor("http://localhost", next_url_generators=[def_rule, generation_rule])
     url = url_extractor.get_next_url()
     def_rule.apply_rule_to.assert_called_with("http://localhost")
     generation_rule.apply_rule_to.assert_called_with("http://localhost/2020")
     self.assertEquals("http://localhost?limit=5&offset=10", url)
Ejemplo n.º 2
0
 def test_get_next_url_should_return_url_from_data_passed_in_if_descriptor_is_list(
         self):
     url_extractor = UrlExtractor(['links', 'self', 'url'])
     self.assertEquals(
         'http://localhost',
         url_extractor.get_next_url(
             {'links': {
                 'self': {
                     'url': 'http://localhost'
                 }
             }}))
Ejemplo n.º 3
0
 def test_get_next_url_should_return_url_using_a_list_of_descriptors(self):
     url_extractor = UrlExtractor([['links', 'other', 'url'],
                                   ['links', 'self', 'url']])
     self.assertEquals(
         'http://localhost',
         url_extractor.get_next_url(
             {'links': {
                 'self': {
                     'url': 'http://localhost'
                 }
             }}))
Ejemplo n.º 4
0
 def test_get_next_url_should_call_apply_all_rules_on_url(self):
     def_rule = Mock(spec=DefaultRule)
     def_rule.apply_rule_to.return_value = 'http://localhost/2020'
     generation_rule = Mock(spec=LimitOffset)
     generation_rule.apply_rule_to.return_value = 'http://localhost?limit=5&offset=10'
     url_extractor = UrlExtractor(
         'http://localhost',
         next_url_generators=[def_rule, generation_rule])
     url = url_extractor.get_next_url()
     def_rule.apply_rule_to.assert_called_with('http://localhost')
     generation_rule.apply_rule_to.assert_called_with(
         'http://localhost/2020')
     self.assertEquals('http://localhost?limit=5&offset=10', url)
Ejemplo n.º 5
0
 def from_url(self,
              url,
              discard=None,
              data_found_at=None,
              next_url_generators=[]):
     self._url_extractor = UrlExtractor(url, next_url_generators)
     self._data_sanitizer = DataSanitizer(discard, data_found_at)
     self._informer = Informer(self._username, self._password)
     return self
Ejemplo n.º 6
0
    def test_get_next_url_should_return_none_in_case_of_error(self):
        url_extractor = UrlExtractor(['links', 'other', 'url'])
        self.assertIsNone(
            url_extractor.get_next_url(
                {'links': {
                    'self': {
                        'url': 'http://localhost'
                    }
                }}))

        url_extractor = UrlExtractor([['links', 'other', 'url'],
                                      ['self', 'other', 'urllib2']])
        self.assertIsNone(
            url_extractor.get_next_url(
                {'links': {
                    'self': {
                        'url': 'http://localhost'
                    }
                }}))
Ejemplo n.º 7
0
 def test_get_next_url_should_return_url_if_set(self):
     url_extractor = UrlExtractor('http://localhost')
     self.assertEquals('http://localhost', url_extractor.get_next_url())
Ejemplo n.º 8
0
 def test_shouldnt_accept_anything_other_than_list_or_string_for_url_descriptor_rest_can_be(
         self):
     with self.assertRaises(AssertionError):
         UrlExtractor({'hello': 'world'})
Ejemplo n.º 9
0
 def test_shouldaccept_string_or_list(self):
     url_extractor = UrlExtractor('http://localhost')
     self.assertEquals('http://localhost', url_extractor._url_descriptor)
     url_extractor = UrlExtractor(['hello', 'world'])
     self.assertEquals(['hello', 'world'], url_extractor._url_descriptor)
Ejemplo n.º 10
0
 def test_get_next_url_should_throw_error_if_data_passed_in_is_none_and_descriptor_is_list(
         self):
     url_extractor = UrlExtractor(['links', 'self', 'url'])
     with self.assertRaises(AssertionError):
         url_extractor.get_next_url()
 def test_get_next_url_should_return_url_using_a_list_of_descriptors(self):
     url_extractor = UrlExtractor([["links", "other", "url"], ["links", "self", "url"]])
     self.assertEquals(
         "http://localhost", url_extractor.get_next_url({"links": {"self": {"url": "http://localhost"}}})
     )
 def test_get_next_url_should_throw_error_if_data_passed_in_is_none_and_descriptor_is_list(self):
     url_extractor = UrlExtractor(["links", "self", "url"])
     with self.assertRaises(AssertionError):
         url_extractor.get_next_url()
 def test_get_next_url_should_return_url_from_data_passed_in_if_descriptor_is_list(self):
     url_extractor = UrlExtractor(["links", "self", "url"])
     self.assertEquals(
         "http://localhost", url_extractor.get_next_url({"links": {"self": {"url": "http://localhost"}}})
     )
 def test_get_next_url_should_return_url_if_set(self):
     url_extractor = UrlExtractor("http://localhost")
     self.assertEquals("http://localhost", url_extractor.get_next_url())
    def test_get_next_url_should_return_none_in_case_of_error(self):
        url_extractor = UrlExtractor(["links", "other", "url"])
        self.assertIsNone(url_extractor.get_next_url({"links": {"self": {"url": "http://localhost"}}}))

        url_extractor = UrlExtractor([["links", "other", "url"], ["self", "other", "urllib2"]])
        self.assertIsNone(url_extractor.get_next_url({"links": {"self": {"url": "http://localhost"}}}))