class StallMan(data.Item):
"""data from stallman.org"""
urgent_items = data.TextField(repeated=True, selector=".column1 li")
class Meta:
base_url = "https://stallman.org/"
fetcher = PhatomJSFetcher
class YifyMovie(data.Item):
details = data.SubPageFields(
MovieDetails, link_selector=".browse-movie-wrap a.browse-movie-link")
pixel = data.TextField(selector=".browse-movie-tags a")
class Meta:
base_url = "https://yts.ag/"
fetcher = fetcher.PhatomJSFetcher
max_workers = 10
class StallMan(data.Item):
urgent_items = data.TextField(repeated=True, selector=".column1 li")
class Meta:
base_url = "https://stallman.org/"
fetcher = fetcher.PhantomProxyFetcher
proxy_list = [
"202.168.244.106:53281", "115.178.97.70:63909",
"203.189.141.162:63909"
]
class MovieDetails(data.Item):
movie_name = data.TextField(selector=".hidden-xs h1")
movie_year = data.TextField(selector=".hidden-xs h2")
def test_get_repeated_bs4selector(self):
"""testing the get value if repeated is enabled"""
field = data.TextField(selector="li", repeated=True)
self.assertListEqual(["1", "2"], field.get_value(self._html))
def test_get_value_for_bs4selector(self):
"""testing the get value if bs4 selector is given"""
field = data.TextField(selector="li")
self.assertEqual("1", field.get_value(self._html))
def test_get_value_for_noneselector(self):
"""testing the get value to be returning none"""
field = data.TextField()
self.assertIsNone(field.get_value(""))