def test_scrapeAppsWithExoticPermissions(): """ Scrape apps with no permissions, only other permissions, etc. Make sure scraper can correctly handle them. :return: """ testCases = { 'kr.co.kcp.wechatpaycheckout', 'com.om.calc', 'com.appdevgenie.electronicscalculator', 'com.androidapps.unitconverter', 'com.videos.freemusic.song.mp3.musicplayer.mv' } dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3') if os.path.exists(dbFilePath): os.remove(dbFilePath) testUtils.runScraper(['--pytest', '-p', ','.join(testCases)]) assert os.path.exists(dbFilePath), "Database file is not created." connection = sqlite3.connect(dbFilePath) cursor = connection.cursor() cursor.execute('select id from App') s = set([r[0] for r in cursor.fetchall()]) failedCases = testCases - s if len(failedCases) > 0: pytest.fail('Failed to scrape ' + ', '.join(failedCases))
def callback_searchResultUpperBound(websiteUrl): dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3') connection = sqlite3.connect(dbFilePath) cursor = connection.cursor() try: appAccessor = GooglePlayAdvancedSearch.DBUtils.AppAccessor() insertedCount = GooglePlayAdvancedSearch.DBUtils.MAX_SELECT + 1 for i in range(insertedCount): app = AppItem() app['id'] = 'GooglePlayAdvancedSearch.testApp' + str(i) app['appName'] = 'matched keyword' app['rating'] = 0 app['install_fee'] = 0 app['app_icon'] = '' appAccessor.insertOrUpdateApp(app) del appAccessor cursor.execute("select count(*) from App where id like 'GooglePlayAdvancedSearch.testApp%'") assert int(cursor.fetchone()[0]) >= insertedCount, f"failed to insert {insertedCount} rows." response = requests.get(websiteUrl + '/Api/Search?q=matched%20keyword', verify=True) data = response.json() assert len(data['apps']) <= GooglePlayAdvancedSearch.DBUtils.MAX_SELECT, f"At most returns {GooglePlayAdvancedSearch.DBUtils.MAX_SELECT}, actually returns {len(data['apps'])}." finally: cursor.execute('delete from App where id like :id', {'id': 'GooglePlayAdvancedSearch.testApp%'})
def websiteUrlCore(): try: response = requests.get('http://localhost:8000', verify=True) yield 'http://localhost:8000' return except requests.exceptions.ConnectionError as e: pass if sys.platform == 'win32': args = ['python', 'manage.py'] else: args = ['./manage.py'] args.extend(['runserver', '--noreload', '8090']) if 'PYTHONIOENCODING' not in os.environ or os.environ[ 'PYTHONIOENCODING'] != 'UTF8': os.environ['PYTHONIOENCODING'] = 'UTF8' p = None try: p = subprocess.Popen(args, cwd=os.path.join(testUtils.getTestFolder(), '../django')) print('web server pid=' + str(p.pid)) time.sleep(1) waitTime = 2 while True: try: response = requests.get('http://localhost:8090', verify=True) yield 'http://localhost:8090' break except requests.exceptions.ConnectionError as e: if waitTime > 10: pytest.fail('Server is not up:\n' + str(e)) time.sleep(waitTime) waitTime = waitTime * waitTime finally: if p: try: parent = psutil.Process(p.pid) for child in parent.children(recursive=True): child.kill() parent.kill() except Exception as e: print('Failed to kill web server processes.\n' + str(e))
def callback_searchCategoryFilter(websiteUrl): # com.facebook.katana uses category 'Social' # we exclude this category in the search, and make sure the result doesn't have com.facebook.katana. testUtils.runScraper(['--pytest', '-p', 'com.facebook.katana']) dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3') connection = sqlite3.connect(dbFilePath) cursor = connection.cursor() categories = GooglePlayAdvancedSearch.DBUtils.getAllCategories(cursor) cid = next((k for k, v in categories.items() if 'Social' in v), None) response = requests.get(websiteUrl + '/Api/Search?q=facebook&cids=' + str(cid), verify=True) text = response.text assert 'com.facebook.katana' not in text, "Search for facebook without Social category. The search result should not have it." response = requests.get(websiteUrl + '/Api/Search?q=facebook', verify=True) text = response.text assert 'com.facebook.katana' in text, "Search for facebook allowing Social category. The search result should have it."
def callback_searchPermissionFilter(websiteUrl): # com.tencent.mm uses permission 'read the contents of your USB storage' # We exclude this permission in the search, and make sure the result doesn't have com.tencent.mm. testUtils.runScraper(['--pytest', '-p', 'com.tencent.mm']) dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3') connection = sqlite3.connect(dbFilePath) cursor = connection.cursor() permissions = GooglePlayAdvancedSearch.DBUtils.getAllPermissions(cursor) pid = next((k for k, v in permissions.items() if 'read the contents of your USB storage' in v), None) response = requests.get(websiteUrl + '/Api/Search?q=wechat&pids=' + str(pid), verify=True) text = response.text assert 'com.tencent.mm' not in text, "Search for wechat without storage permission. The search result should not have wechat." response = requests.get(websiteUrl + '/Api/Search?q=wechat', verify=True) text = response.text assert 'com.tencent.mm' in text, "Search for wechat allowing storage permission. The search result not have wechat."
def dbFilePath(): return os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3')