def checkOneApp(self, apkid): """ checkOneApp(apkid): Collect all versions for an application """ logging.info('Checking app: {0}'.format(apkid)) filenames = [] try: apkMirrorName = allApkMirrorNames[apkid] # Using the "uploads/?q=" page sorts newest first but is slower # Using the "apk/google-inc/" page is faster loading # For now favor slow load and skip checking all versions (below) url = 'http://www.apkmirror.com/uploads/?q={0}'.format( apkMirrorName) session = requests.Session() logging.debug('Requesting1: ' + url) try: resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode( 'ascii', 'ignore') dom = BeautifulSoup(html, 'html5lib') versions = dom.findAll('div', {'class': 'infoSlide'}) hasVariants = dom.find('svg', {'class': 'tag-icon'}) avis = [] # Skip all version since it is sorted (above) # for version in versions: version = versions[0] verName = version.findAll('span', {'class': 'infoslide-value'}) verName = verName[0].get_text() appNameRow = version.find_previous_sibling( 'div', {'class': 'appRow'}) appNameLink = appNameRow.find('a', {'class': 'fontBlack'}) appName = appNameLink.get_text() appUrl = appNameLink['href'] if 'preview' in appName.lower(): logging.info('!!! Preview Found: ' + appName) else: isBeta = 'beta' in appName.lower() avi = ApkVersionInfo(name=apkid + ('.beta' if isBeta else ''), ver=verName, scrape_src=appUrl) if self.report.isThisApkNeeded(avi): if hasVariants: avis.extend(self.getMultipleVersionInfo(avi)) else: tmp = self.getOneVersionInfo(avi) if tmp: avis.append(tmp) # END: for version in versions: # Determine which versions to download for avi in avis: if self.report.isThisApkNeeded(avi): logging.info('Downloading: "{0}"'.format( avi.getFilename())) filenames.append( self.downloadApk(avi, avi.name.endswith('.beta'))) else: logging.debug('Skipping: "{0}" ({1})'.format( avi.name, avi.scrape_src)) # END: for avi in avis: except: logging.exception( '!!! Error parsing html from: "{0}"'.format(url)) except KeyError: logging.info('{0} not in apkmirror.com dictionary'.format(apkid)) return filenames
def checkOneApp(self, apkid): """ checkOneApp(apkid): Collect all versions for an application """ logging.info('Checking app: {0}'.format(apkid)) filenames = [] try: apkMirrorName = allApkMirrorNames[apkid] # Using the "uploads/?q=" page sorts newest first but is slower # Using the "apk/google-inc/" page is faster loading # For now favor slow load and skip checking all versions (below) url = 'http://www.apkmirror.com/uploads/?q={0}'.format(apkMirrorName) session = requests.Session() session.headers.update(self.headers) logging.debug('Requesting1: ' + url) try: resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore') dom = BeautifulSoup(html, 'html5lib') versions = dom.findAll('div', {'class': 'infoSlide'}) hasVariants = dom.find('svg', {'class': 'tag-icon'}) avis = [] # Skip all version since it is sorted (above) # for version in versions: version = versions[0] verName = version.findAll('span', {'class': 'infoslide-value'}) verName = verName[0].get_text() appNameRow = version.find_previous_sibling('div', {'class': 'appRow'}) appNameLink = appNameRow.find('a', {'class': 'fontBlack'}) appName = appNameLink.get_text() appUrl = appNameLink['href'] if 'preview' in appName.lower(): logging.info('!!! Preview Found: ' + appName) else: isBeta = 'beta' in appName.lower() avi = ApkVersionInfo(name=apkid + ('.beta' if isBeta else ''), ver=verName, scrape_src=appUrl) if self.report.isThisApkNeeded(avi): if hasVariants: avis.extend(self.getMultipleVersionInfo(avi)) else: tmp = self.getOneVersionInfo(avi) if tmp: avis.append(tmp) # END: for version in versions: # Determine which versions to download for avi in avis: if self.report.isThisApkNeeded(avi): logging.info('Downloading: "{0}"'.format(avi.getFilename())) filenames.append(self.downloadApk(avi, avi.name.endswith('.beta'))) else: logging.debug('Skipping: "{0}" ({1})'.format(avi.name, avi.scrape_src)) # END: for avi in avis: except: logging.exception('!!! Error parsing html from: "{0}"'.format(url)) except KeyError: logging.info('{0} not in apkmirror.com dictionary'.format(apkid)) return filenames