def test_scrapeAppsWithExoticPermissions():
"""
Scrape apps with no permissions, only other permissions, etc.
Make sure scraper can correctly handle them.
:return:
"""
testCases = {
'kr.co.kcp.wechatpaycheckout', 'com.om.calc',
'com.appdevgenie.electronicscalculator',
'com.androidapps.unitconverter',
'com.videos.freemusic.song.mp3.musicplayer.mv'
}
dbFilePath = os.path.join(testUtils.getTestFolder(),
'../../data/db.sqlite3')
if os.path.exists(dbFilePath):
os.remove(dbFilePath)
testUtils.runScraper(['--pytest', '-p', ','.join(testCases)])
assert os.path.exists(dbFilePath), "Database file is not created."
connection = sqlite3.connect(dbFilePath)
cursor = connection.cursor()
cursor.execute('select id from App')
s = set([r[0] for r in cursor.fetchall()])
failedCases = testCases - s
if len(failedCases) > 0:
pytest.fail('Failed to scrape ' + ', '.join(failedCases))
def callback_searchResultUpperBound(websiteUrl):
dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3')
connection = sqlite3.connect(dbFilePath)
cursor = connection.cursor()
try:
appAccessor = GooglePlayAdvancedSearch.DBUtils.AppAccessor()
insertedCount = GooglePlayAdvancedSearch.DBUtils.MAX_SELECT + 1
for i in range(insertedCount):
app = AppItem()
app['id'] = 'GooglePlayAdvancedSearch.testApp' + str(i)
app['appName'] = 'matched keyword'
app['rating'] = 0
app['install_fee'] = 0
app['app_icon'] = ''
appAccessor.insertOrUpdateApp(app)
del appAccessor
cursor.execute("select count(*) from App where id like 'GooglePlayAdvancedSearch.testApp%'")
assert int(cursor.fetchone()[0]) >= insertedCount, f"failed to insert {insertedCount} rows."
response = requests.get(websiteUrl + '/Api/Search?q=matched%20keyword', verify=True)
data = response.json()
assert len(data['apps']) <= GooglePlayAdvancedSearch.DBUtils.MAX_SELECT, f"At most returns {GooglePlayAdvancedSearch.DBUtils.MAX_SELECT}, actually returns {len(data['apps'])}."
finally:
cursor.execute('delete from App where id like :id', {'id': 'GooglePlayAdvancedSearch.testApp%'})
def websiteUrlCore():
try:
response = requests.get('http://localhost:8000', verify=True)
yield 'http://localhost:8000'
return
except requests.exceptions.ConnectionError as e:
pass
if sys.platform == 'win32':
args = ['python', 'manage.py']
else:
args = ['./manage.py']
args.extend(['runserver', '--noreload', '8090'])
if 'PYTHONIOENCODING' not in os.environ or os.environ[
'PYTHONIOENCODING'] != 'UTF8':
os.environ['PYTHONIOENCODING'] = 'UTF8'
p = None
try:
p = subprocess.Popen(args,
cwd=os.path.join(testUtils.getTestFolder(),
'../django'))
print('web server pid=' + str(p.pid))
time.sleep(1)
waitTime = 2
while True:
try:
response = requests.get('http://localhost:8090', verify=True)
yield 'http://localhost:8090'
break
except requests.exceptions.ConnectionError as e:
if waitTime > 10:
pytest.fail('Server is not up:\n' + str(e))
time.sleep(waitTime)
waitTime = waitTime * waitTime
finally:
if p:
try:
parent = psutil.Process(p.pid)
for child in parent.children(recursive=True):
child.kill()
parent.kill()
except Exception as e:
print('Failed to kill web server processes.\n' + str(e))
def callback_searchCategoryFilter(websiteUrl): # com.facebook.katana uses category 'Social' # we exclude this category in the search, and make sure the result doesn't have com.facebook.katana. testUtils.runScraper(['--pytest', '-p', 'com.facebook.katana']) dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3') connection = sqlite3.connect(dbFilePath) cursor = connection.cursor() categories = GooglePlayAdvancedSearch.DBUtils.getAllCategories(cursor) cid = next((k for k, v in categories.items() if 'Social' in v), None) response = requests.get(websiteUrl + '/Api/Search?q=facebook&cids=' + str(cid), verify=True) text = response.text assert 'com.facebook.katana' not in text, "Search for facebook without Social category. The search result should not have it." response = requests.get(websiteUrl + '/Api/Search?q=facebook', verify=True) text = response.text assert 'com.facebook.katana' in text, "Search for facebook allowing Social category. The search result should have it."
def callback_searchPermissionFilter(websiteUrl): # com.tencent.mm uses permission 'read the contents of your USB storage' # We exclude this permission in the search, and make sure the result doesn't have com.tencent.mm. testUtils.runScraper(['--pytest', '-p', 'com.tencent.mm']) dbFilePath = os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3') connection = sqlite3.connect(dbFilePath) cursor = connection.cursor() permissions = GooglePlayAdvancedSearch.DBUtils.getAllPermissions(cursor) pid = next((k for k, v in permissions.items() if 'read the contents of your USB storage' in v), None) response = requests.get(websiteUrl + '/Api/Search?q=wechat&pids=' + str(pid), verify=True) text = response.text assert 'com.tencent.mm' not in text, "Search for wechat without storage permission. The search result should not have wechat." response = requests.get(websiteUrl + '/Api/Search?q=wechat', verify=True) text = response.text assert 'com.tencent.mm' in text, "Search for wechat allowing storage permission. The search result not have wechat."
def dbFilePath():
return os.path.join(testUtils.getTestFolder(), '../../data/db.sqlite3')