def checkOneApp(self, apkid):
"""
checkOneApp(apkid): Collect all versions for an application
"""
logging.info('Checking app: {0}'.format(apkid))
filenames = []
try:
apkMirrorName = allApkMirrorNames[apkid]
# Using the "uploads/?q=" page sorts newest first but is slower
# Using the "apk/google-inc/" page is faster loading
# For now favor slow load and skip checking all versions (below)
url = 'http://www.apkmirror.com/uploads/?q={0}'.format(
apkMirrorName)
session = requests.Session()
logging.debug('Requesting1: ' + url)
try:
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode(
'ascii', 'ignore')
dom = BeautifulSoup(html, 'html5lib')
versions = dom.findAll('div', {'class': 'infoSlide'})
hasVariants = dom.find('svg', {'class': 'tag-icon'})
avis = []
# Skip all version since it is sorted (above)
# for version in versions:
version = versions[0]
verName = version.findAll('span', {'class': 'infoslide-value'})
verName = verName[0].get_text()
appNameRow = version.find_previous_sibling(
'div', {'class': 'appRow'})
appNameLink = appNameRow.find('a', {'class': 'fontBlack'})
appName = appNameLink.get_text()
appUrl = appNameLink['href']
if 'preview' in appName.lower():
logging.info('!!! Preview Found: ' + appName)
else:
isBeta = 'beta' in appName.lower()
avi = ApkVersionInfo(name=apkid +
('.beta' if isBeta else ''),
ver=verName,
scrape_src=appUrl)
if self.report.isThisApkNeeded(avi):
if hasVariants:
avis.extend(self.getMultipleVersionInfo(avi))
else:
tmp = self.getOneVersionInfo(avi)
if tmp:
avis.append(tmp)
# END: for version in versions:
# Determine which versions to download
for avi in avis:
if self.report.isThisApkNeeded(avi):
logging.info('Downloading: "{0}"'.format(
avi.getFilename()))
filenames.append(
self.downloadApk(avi, avi.name.endswith('.beta')))
else:
logging.debug('Skipping: "{0}" ({1})'.format(
avi.name, avi.scrape_src))
# END: for avi in avis:
except:
logging.exception(
'!!! Error parsing html from: "{0}"'.format(url))
except KeyError:
logging.info('{0} not in apkmirror.com dictionary'.format(apkid))
return filenames
def checkOneApp(self, apkid):
"""
checkOneApp(apkid): Collect all versions for an application
"""
logging.info('Checking app: {0}'.format(apkid))
filenames = []
try:
apkMirrorName = allApkMirrorNames[apkid]
# Using the "uploads/?q=" page sorts newest first but is slower
# Using the "apk/google-inc/" page is faster loading
# For now favor slow load and skip checking all versions (below)
url = 'http://www.apkmirror.com/uploads/?q={0}'.format(apkMirrorName)
session = requests.Session()
session.headers.update(self.headers)
logging.debug('Requesting1: ' + url)
try:
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
dom = BeautifulSoup(html, 'html5lib')
versions = dom.findAll('div', {'class': 'infoSlide'})
hasVariants = dom.find('svg', {'class': 'tag-icon'})
avis = []
# Skip all version since it is sorted (above)
# for version in versions:
version = versions[0]
verName = version.findAll('span', {'class': 'infoslide-value'})
verName = verName[0].get_text()
appNameRow = version.find_previous_sibling('div', {'class': 'appRow'})
appNameLink = appNameRow.find('a', {'class': 'fontBlack'})
appName = appNameLink.get_text()
appUrl = appNameLink['href']
if 'preview' in appName.lower():
logging.info('!!! Preview Found: ' + appName)
else:
isBeta = 'beta' in appName.lower()
avi = ApkVersionInfo(name=apkid + ('.beta' if isBeta else ''),
ver=verName,
scrape_src=appUrl)
if self.report.isThisApkNeeded(avi):
if hasVariants:
avis.extend(self.getMultipleVersionInfo(avi))
else:
tmp = self.getOneVersionInfo(avi)
if tmp:
avis.append(tmp)
# END: for version in versions:
# Determine which versions to download
for avi in avis:
if self.report.isThisApkNeeded(avi):
logging.info('Downloading: "{0}"'.format(avi.getFilename()))
filenames.append(self.downloadApk(avi, avi.name.endswith('.beta')))
else:
logging.debug('Skipping: "{0}" ({1})'.format(avi.name, avi.scrape_src))
# END: for avi in avis:
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
except KeyError:
logging.info('{0} not in apkmirror.com dictionary'.format(apkid))
return filenames
