def checkOneApp(self, apkid):
"""
checkOneApp(apkid):
"""
logging.info('Checking app: {0}'.format(apkid))
filenames = []
url = 'http://apk-dl.com/' + apkid
session = requests.Session()
logging.debug('Requesting: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
try:
dom = BeautifulSoup(html, 'html5lib')
apklist = dom.findAll('ul', {'class': 'apks dlist'})[0]
apks = apklist.findAll('div', {'class': 'details'})
for apk in apks:
items = apk.findAll('div')
dApk = {}
for item in items:
itext = '{0}'.format(item.get_text().encode('ascii', 'ignore'))
itext = re.sub('\s', '', itext)
itextsp = itext.split(':', 1)
if len(itextsp) == 2:
dApk[str(itextsp[0])] = str(itextsp[1])
apkurl = apk.find('a', {'class': 'btn btn-success'})
if apkurl:
dApk['url'] = 'http:' + apkurl['href']
Debug.printDictionary(dApk)
if 'Version' in dApk and 'RequiresAndroid' in dApk:
(trash, sdk) = dApk['RequiresAndroid'].split('API:', 1)
sdk = sdk[0:-1]
(ver, vercode) = dApk['Version'].split('(Code:', 1)
ver = ver.split('(', 1)[0].strip()
vercode = vercode[0:-1].strip()
avi = ApkVersionInfo(name=apkid,
sdk=sdk,
ver=ver,
vercode=vercode,
download_src=dApk['url'],
crawler_name=self.__class__.__name__
)
if self.report.isThisApkNeeded(avi):
filenames.append(self.downloadApk(avi))
except IndexError:
logging.info('{0} not supported by apk-dl.com ...'.format(apkid))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
return filenames
def getAppVersions(apkInfo):
"""
getAppVersions(apkInfo): Collect all versions for an applicaiton
"""
logging.info('Fetching Information for: {0}'.format(apkInfo.apkmirror_name))
html_name = '{0}.html'.format(apkInfo.opengapps_name)
url = APKMIRRORBASEURL + APKMIRRORGOOGLEURL2 + apkInfo.url
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
logging.debug('Requesting2: ' + url)
resp = session.get(url)
html = resp.text
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
latest = dom.findAll('div', {'class': 'latestWidget'})[1]
versions = latest.findAll('a', {'class': 'fontBlack'})
dVersions = {}
for version in versions:
# Ignore duplicate entries (one 'hidden' is shown,
# and the other 'visible', is not shown)
if 'visible-xs-block' in version.parent['class']:
continue
verText = '"{0}"'.format(version.get_text().encode('ascii', 'ignore'))
if 'beta' in verText.lower() or 'preview' in verText.lower():
logging.info('!!! Beta or Preview Found: ' + verText)
else:
dVersions[verText] = version['href']
m = apkInfo.reVersion.search(verText)
if m:
avi = ApkVersionInfo(name=m.group('VERSIONNAME').rstrip('-.'), scrape_url=version['href'])
avi.ver = avi.name
avi.ver = avi.ver.replace(apkInfo.opengapps_name, '').strip()
avi.ver = avi.ver.split(' ')[0]
apkInfo.versions.append(avi)
else:
logging.info('!!! No Matchy: ' + verText)
# END: for v in versions:
Debug.printDictionary(dVersions)
# Determine which versions to download
if len(apkInfo.versions) > 0:
maxVersionByName = sorted(apkInfo.versions)[-1]
logging.debug('Max Version By Name: "{0}"'.format(maxVersionByName.name))
for v in apkInfo.versions:
if v.name == maxVersionByName.name:
logging.info('Getting Info for: "{0}" ({1})'.format(v.name, v.scrape_url))
getVersionInfo(v)
logging.info('Downloading: "{0}"'.format(v.apk_name))
downloadApkFromVersionInfo(v)
else:
logging.debug('Skipping: "{0}" ({1})'.format(v.name, v.scrape_url))
# END: for v in apkInfo.versions:
else:
logging.info('No matching APKs found for: {0}'.format(apkInfo.apkmirror_name))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
logging.debug('-'*80)
def checkOneApp(apkid):
"""
checkOneApp(apkid):
"""
dAllApks = Global.dAllApks
maxVerEachApk = Global.maxVerEachApk
minSdkEachApk = Global.minSdkEachApk
logging.info('Checking app: {0}'.format(apkid))
html_name = '{0}.html'.format(apkid)
url = 'http://apk-dl.com/' + apkid
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
session.proxies = Debug.getProxy()
logging.debug('Requesting: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
apklist = dom.findAll('ul', {'class': 'apks dlist'})[0]
apks = apklist.findAll('div', {'class': 'details'})
maxApkInfo = ApkVersionInfo(name=apkid, ver=maxVerEachApk[apkid])
for apk in apks:
items = apk.findAll('div')
dApk = {}
for item in items:
itext = '{0}'.format(item.get_text().encode('ascii', 'ignore'))
itext = re.sub('\s', '', itext)
itextsp = itext.split(':', 1)
if len(itextsp) == 2:
dApk[str(itextsp[0])] = str(itextsp[1])
dApk['url'] = 'http:' + apk.find('a', {'class': 'btn btn-success'})['href']
Debug.printDictionary(dApk)
if 'Version' in dApk and 'RequiresAndroid' in dApk:
(trash, sdk) = dApk['RequiresAndroid'].split('API:', 1)
sdk = sdk[0:-1]
(ver, vercode) = dApk['Version'].split('(Code:', 1)
ver = ver.split('(', 1)[0]
vercode = vercode[0:-1]
tmpApkInfo = ApkVersionInfo(name=apkid, sdk=sdk, ver=ver, vercode=vercode)
tmpApkInfo.download_url = dApk['url']
if maxApkInfo <= tmpApkInfo:
thisSdk = int(tmpApkInfo.sdk)
if thisSdk < minSdkEachApk[apkid]:
logging.debug('SdkTooLow: {0}({1})'.format(apkid, thisSdk))
continue
if not filter(lambda apk: apk.vercode == tmpApkInfo.vercode, dAllApks[apkid]):
logging.debug(tmpApkInfo.fullString(maxVerEachApk[apkid]))
downloadApk(tmpApkInfo)
except IndexError:
logging.info('{0} not supported by apk-dl.com ...'.format(apkid))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
