def test_non_detailed_different_language_and_country(self): s = PlayScraper(hl="da", gl="dk") apps = s.collection("TOP_PAID", "LIFESTYLE", results=5) self.assertEqual(5, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl="da", gl="dk") apps = s.developer("Google LLC", results=5) self.assertEqual(5, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl="da", gl="dk") apps = s.search("dog") self.assertEqual(50, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') apps = s.search('dog') self.assertEqual(20, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl="da", gl="dk") apps = s.similar("com.android.chrome") self.assertGreater(len(apps), 0) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') apps = s.developer('Google LLC', results=5) self.assertEqual(5, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_non_detailed_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') apps = s.collection('TOP_PAID', 'LIFESTYLE', results=5) self.assertEqual(5, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') apps = s.collection('TOP_PAID', 'LIFESTYLE', results=5) self.assertEqual(5, len(apps)) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') apps = s.similar('com.android.chrome') self.assertGreater(len(apps), 0) self.assertTrue(all(key in apps[0] for key in BASIC_KEYS)) self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_fetching_app_in_spanish(self): s = PlayScraper(hl='es', gl='es') app_data = s.details('com.android.chrome') self.assertTrue(all(key in app_data for key in DETAIL_KEYS)) self.assertEqual(len(DETAIL_KEYS), len(app_data.keys())) self.assertEqual('com.android.chrome', app_data['app_id']) self.assertEqual(['COMMUNICATION'], app_data['category']) # additional details are all None because we currently hardcode an # English mapping for the various additional info section titles. self.assertTrue(all([app_data[x] is None for x in ADDITIONAL_INFO_KEYS]))
def test_detailed_collection_different_language(self): s = PlayScraper(hl="da", gl="dk") apps = s.collection("TOP_FREE", results=1, detailed=True) self.assertEqual(1, len(apps)) self.assertTrue(all(key in apps[0] for key in DETAIL_KEYS)) self.assertEqual(len(DETAIL_KEYS), len(apps[0].keys())) # additional details, like installs, are all None because we currently # hardcode an English mapping for the various additional info section # titles. self.assertTrue(all([apps[0][x] is None for x in ADDITIONAL_INFO_KEYS]))
def test_detailed_collection_different_language(self): apps_default = self.s.collection('TOP_FREE', results=1, detailed=True) s = PlayScraper(hl='da', gl='dk') apps = s.collection('TOP_FREE', results=1, detailed=True) self.assertEqual(1, len(apps)) self.assertTrue(all(key in apps[0] for key in DETAIL_KEYS)) self.assertEqual(len(DETAIL_KEYS), len(apps[0].keys())) # additional details, like installs, are all None because we currently # hardcode an English mapping for the various additional info section # titles. self.assertTrue(all([apps[0][x] is None for x in ADDITIONAL_INFO_KEYS]))
def test_fetching_app_in_spanish(self): s = PlayScraper(hl='es', gl='es') app_data = s.details('com.android.chrome') self.assertTrue(all(key in app_data for key in DETAIL_KEYS)) self.assertEqual(len(DETAIL_KEYS), len(app_data.keys())) self.assertEqual('com.android.chrome', app_data['app_id']) self.assertEqual(['COMMUNICATION'], app_data['category']) self.assertTrue(all(x is not None and x.startswith('https://') for x in app_data['screenshots'])) # additional details, like installs, are all None because we currently # hardcode an English mapping for the various additional info section # titles. self.assertTrue(all([app_data[x] is None for x in ADDITIONAL_INFO_KEYS]))
def test_init_with_language_and_geolocation(self): s = PlayScraper(hl='ko', gl='kr') self.assertEqual('ko', s.language) self.assertEqual('kr', s.geolocation) self.assertDictEqual({'hl': 'ko', 'gl': 'kr'}, s.params)
def test_fetching_app_in_spanish(self): s = PlayScraper(hl="es", gl="es") app_data = s.details("com.android.chrome") self.assertTrue(all(key in app_data for key in DETAIL_KEYS)) self.assertEqual(len(DETAIL_KEYS), len(app_data.keys())) self.assertEqual("com.android.chrome", app_data["app_id"]) self.assertEqual(["COMMUNICATION"], app_data["category"]) self.assertTrue(app_data["screenshots"] and all(x is not None and x.startswith("https://") for x in app_data["screenshots"])) # additional details, like installs, are all None because we currently # hardcode an English mapping for the various additional info section # titles. self.assertTrue( all([app_data[x] is None for x in ADDITIONAL_INFO_KEYS]))
def test_init_with_defaults(self): s = PlayScraper() self.assertEqual(settings.BASE_URL, s._base_url) self.assertEqual(settings.SUGGESTION_URL, s._suggestion_url) self.assertEqual(settings.SEARCH_URL, s._search_url) self.assertEqual(settings.PAGE_TOKENS, s._pagtok) self.assertEqual("en", s.language) self.assertEqual("us", s.geolocation) self.assertDictEqual({"hl": "en", "gl": "us"}, s.params)
def test_init_with_defaults(self): s = PlayScraper() self.assertEqual(settings.BASE_URL, s._base_url) self.assertEqual(settings.SUGGESTION_URL, s._suggestion_url) self.assertEqual(settings.SEARCH_URL, s._search_url) self.assertEqual(settings.PAGE_TOKENS, s._pagtok) self.assertEqual('en', s.language) self.assertEqual('us', s.geolocation) self.assertDictEqual({'hl': 'en', 'gl': 'us'}, s.params)
def setUp(self): self.s = PlayScraper()
def test_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') suggestions = s.suggestions('dog') self.assertGreater(len(suggestions), 0)
def test_different_language_and_country(self): s = PlayScraper(hl="da", gl="dk") categories = s.categories() self.assertTrue(all(key in categories for key in CATEGORIES))
def test_different_language_and_country(self): s = PlayScraper(hl="da", gl="dk") suggestions = s.suggestions("dog") self.assertGreater(len(suggestions), 0)
def test_init_with_language_and_geolocation(self): s = PlayScraper(hl="ko", gl="kr") self.assertEqual("ko", s.language) self.assertEqual("kr", s.geolocation) self.assertDictEqual({"hl": "ko", "gl": "kr"}, s.params)
import concurrent import json import os from concurrent.futures.thread import ThreadPoolExecutor from datetime import datetime from threading import Lock import play_scraper import play_scraper.settings import play_scraper.utils from play_scraper.scraper import PlayScraper scraper = PlayScraper() play_scraper.settings.CONCURRENT_REQUESTS = 20 base_addr = "data" stats_lock = Lock() stats = { 'details-checked': set(), 'developers-not-checked': set(), 'developers-checked': set(), 'similars-checked': set(), 'similars-not-checked': set(), 'categories-checked': set(), } def load_stats(): log('loading stats....') for file in stats: addr = os.path.join(base_addr, f'stats/{file}.txt')
def test_invalid_geolocation_code_raises(self): with self.assertRaises(ValueError) as e: PlayScraper(gl="invalid") self.assertEqual("invalid is not a valid geolocation country code.", str(e.exception))
def test_invalid_language_code_raises(self): with self.assertRaises(ValueError) as e: PlayScraper(hl="invalid") self.assertEqual("invalid is not a valid language interface code.", str(e.exception))
def test_different_language_and_country(self): s = PlayScraper(hl='da', gl='dk') categories = s.categories() self.assertTrue(all(key in categories for key in CATEGORIES))