def test_non_detailed_different_language_and_country(self):
s = PlayScraper(hl="da", gl="dk")
apps = s.collection("TOP_PAID", "LIFESTYLE", results=5)
self.assertEqual(5, len(apps))
self.assertTrue(all(key in apps[0] for key in BASIC_KEYS))
self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self):
s = PlayScraper(hl="da", gl="dk")
apps = s.similar("com.android.chrome")
self.assertGreater(len(apps), 0)
self.assertTrue(all(key in apps[0] for key in BASIC_KEYS))
self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self):
s = PlayScraper(hl="da", gl="dk")
apps = s.search("dog")
self.assertEqual(50, len(apps))
self.assertTrue(all(key in apps[0] for key in BASIC_KEYS))
self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self):
s = PlayScraper(hl='da', gl='dk')
apps = s.collection('TOP_PAID', 'LIFESTYLE', results=5)
self.assertEqual(5, len(apps))
self.assertTrue(all(key in apps[0] for key in BASIC_KEYS))
self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_different_language_and_country(self):
s = PlayScraper(hl='da', gl='dk')
apps = s.search('dog')
self.assertEqual(20, len(apps))
self.assertTrue(all(key in apps[0] for key in BASIC_KEYS))
self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_init_with_language_and_geolocation(self):
s = PlayScraper(hl='ko', gl='kr')
self.assertEqual('ko', s.language)
self.assertEqual('kr', s.geolocation)
self.assertDictEqual({'hl': 'ko',
'gl': 'kr'},
s.params)
def test_different_language_and_country(self):
s = PlayScraper(hl="da", gl="dk")
apps = s.developer("Google LLC", results=5)
self.assertEqual(5, len(apps))
self.assertTrue(all(key in apps[0] for key in BASIC_KEYS))
self.assertEqual(len(BASIC_KEYS), len(apps[0].keys()))
def test_init_with_defaults(self):
s = PlayScraper()
self.assertEqual(settings.BASE_URL, s._base_url)
self.assertEqual(settings.SUGGESTION_URL, s._suggestion_url)
self.assertEqual(settings.SEARCH_URL, s._search_url)
self.assertEqual(settings.PAGE_TOKENS, s._pagtok)
self.assertEqual("en", s.language)
self.assertEqual("us", s.geolocation)
self.assertDictEqual({"hl": "en", "gl": "us"}, s.params)
def test_init_with_defaults(self):
s = PlayScraper()
self.assertEqual(settings.BASE_URL, s._base_url)
self.assertEqual(settings.SUGGESTION_URL, s._suggestion_url)
self.assertEqual(settings.SEARCH_URL, s._search_url)
self.assertEqual(settings.PAGE_TOKENS, s._pagtok)
self.assertEqual('en', s.language)
self.assertEqual('us', s.geolocation)
self.assertDictEqual({'hl': 'en', 'gl': 'us'}, s.params)
def test_fetching_app_in_spanish(self):
s = PlayScraper(hl='es', gl='es')
app_data = s.details('com.android.chrome')
self.assertTrue(all(key in app_data for key in DETAIL_KEYS))
self.assertEqual(len(DETAIL_KEYS), len(app_data.keys()))
self.assertEqual('com.android.chrome', app_data['app_id'])
self.assertEqual(['COMMUNICATION'], app_data['category'])
# additional details are all None because we currently hardcode an
# English mapping for the various additional info section titles.
self.assertTrue(all([app_data[x] is None
for x in ADDITIONAL_INFO_KEYS]))
def test_detailed_collection_different_language(self):
s = PlayScraper(hl="da", gl="dk")
apps = s.collection("TOP_FREE", results=1, detailed=True)
self.assertEqual(1, len(apps))
self.assertTrue(all(key in apps[0] for key in DETAIL_KEYS))
self.assertEqual(len(DETAIL_KEYS), len(apps[0].keys()))
# additional details, like installs, are all None because we currently
# hardcode an English mapping for the various additional info section
# titles.
self.assertTrue(all([apps[0][x] is None
for x in ADDITIONAL_INFO_KEYS]))
def test_fetching_app_in_spanish(self):
s = PlayScraper(hl="es", gl="es")
app_data = s.details("com.android.chrome")
self.assertTrue(all(key in app_data for key in DETAIL_KEYS))
self.assertEqual(len(DETAIL_KEYS), len(app_data.keys()))
self.assertEqual("com.android.chrome", app_data["app_id"])
self.assertEqual(["COMMUNICATION"], app_data["category"])
self.assertTrue(app_data["screenshots"]
and all(x is not None and x.startswith("https://")
for x in app_data["screenshots"]))
# additional details, like installs, are all None because we currently
# hardcode an English mapping for the various additional info section
# titles.
self.assertTrue(
all([app_data[x] is None for x in ADDITIONAL_INFO_KEYS]))
def test_invalid_language_code_raises(self):
with self.assertRaises(ValueError) as e:
PlayScraper(hl="invalid")
self.assertEqual("invalid is not a valid language interface code.",
str(e.exception))
def test_init_with_language_and_geolocation(self):
s = PlayScraper(hl="ko", gl="kr")
self.assertEqual("ko", s.language)
self.assertEqual("kr", s.geolocation)
self.assertDictEqual({"hl": "ko", "gl": "kr"}, s.params)
def test_different_language_and_country(self):
s = PlayScraper(hl='da', gl='dk')
suggestions = s.suggestions('dog')
self.assertGreater(len(suggestions), 0)
def setUp(self):
self.s = PlayScraper()
def test_different_language_and_country(self):
s = PlayScraper(hl="da", gl="dk")
categories = s.categories()
self.assertTrue(all(key in categories for key in CATEGORIES))
def test_invalid_geolocation_code_raises(self):
with self.assertRaises(ValueError) as e:
PlayScraper(gl="invalid")
self.assertEqual("invalid is not a valid geolocation country code.",
str(e.exception))
def test_different_language_and_country(self):
s = PlayScraper(hl="da", gl="dk")
suggestions = s.suggestions("dog")
self.assertGreater(len(suggestions), 0)
import concurrent
import json
import os
from concurrent.futures.thread import ThreadPoolExecutor
from datetime import datetime
from threading import Lock
import play_scraper
import play_scraper.settings
import play_scraper.utils
from play_scraper.scraper import PlayScraper
scraper = PlayScraper()
play_scraper.settings.CONCURRENT_REQUESTS = 20
base_addr = "data"
stats_lock = Lock()
stats = {
'details-checked': set(),
'developers-not-checked': set(),
'developers-checked': set(),
'similars-checked': set(),
'similars-not-checked': set(),
'categories-checked': set(),
}
def load_stats():
log('loading stats....')
for file in stats:
addr = os.path.join(base_addr, f'stats/{file}.txt')