def listRepo(repo, orderby=None): """ listRepo(repo): Get list of all APKs in a specific store from the Aptoide API and return it in JSON form """ file_name = '{0}{1}.json'.format(repo, '' if not orderby else '-' + '-'.join(orderby)) orderby = '' if not orderby else '/orderby/' + '/'.join(orderby) url = 'http://webservices.aptoide.com/webservices/listRepository/{0}{1}/json'.format(repo, orderby) data = Debug.readFromFile(file_name) try: if data == '': session = requests.Session() # session.proxies = Debug.getProxy() logging.debug('Requesting1: ' + url) resp = session.get(url) data = json.loads(resp.text) Debug.writeToFile(file_name, json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')), resp.encoding) if data['status'] == 'OK': return data else: logging.error(file_name) except: logging.exception('!!! Invalid JSON from: "{0}"'.format(url)) return None
def getVersionInfo(apkVersionInfo): """ getVersionInfo(apkVersionInfo): Determines each versions information """ html_name = apkVersionInfo.scrape_url.rsplit('/', 2)[1] html_name = html_name.replace('-android-apk-download', '') + '.html' url = APKMIRRORBASEURL + apkVersionInfo.scrape_url html = Debug.readFromFile(html_name) if html == '': session = requests.Session() logging.debug('Requesting3: ' + url) resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore') Debug.writeToFile(html_name, html, resp.encoding) try: dom = BeautifulSoup(html, 'html5lib') postArea = dom.findAll('div', {'class': 'post-area'})[0] dl_button = postArea.findAll('a', {'type': 'button'})[1] blueFonts = postArea.findAll('span', {'class': 'fontBlue'}) apkVersionInfo.download_url = dl_button['href'] for blueFont in blueFonts: if blueFont.get_text() == 'File name: ': apkVersionInfo.apk_name = blueFont.next_sibling if blueFont.get_text() == 'Version: ': apkVersionInfo.ver = blueFont.next_sibling except: logging.exception('!!! Error parsing html from: "{0}"'.format(url))
def getApkInfo(repo, apkid, apkversion, options=None, doVersion1=False): """ getApkInfo(repo, apkid, apkversion): Get APK specific information from the Aptoide API and return it in JSON form """ version = '1' if doVersion1 else '2' file_name = '{0}-{1}-{2}_{3}.json'.format(repo, apkid, apkversion, version) options = '' if not options else '/options=({0})'.format(options) url = 'http://webservices.aptoide.com/webservices/{0}/getApkInfo/{1}/{2}/{3}{4}/json'.format( version, repo, apkid, apkversion, options) data = Debug.readFromFile(file_name) try: if data == '': session = requests.Session() # session.proxies = Debug.getProxy() logging.debug('Requesting2: ' + url) resp = session.get(url) data = json.loads(resp.text) Debug.writeToFile(file_name, json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')), resp.encoding) if data['status'] == 'OK': return data else: logging.error(file_name) except: logging.exception('!!! Invalid JSON from: "{0}"'.format(url)) return None
def checkOneApp(apkid): """ checkOneApp(apkid): """ dAllApks = Global.dAllApks maxVerEachApk = Global.maxVerEachApk minSdkEachApk = Global.minSdkEachApk logging.info("Checking app: {0}".format(apkid)) file_name = "{0}.json".format(apkid) url = "http://helper.mgccw.com/nclient/sjson/detail/detailInfo.htm?apkId=" + apkid data = Debug.readFromFile(file_name) try: if data == "": session = requests.Session() # session.proxies = Debug.getProxy() logging.debug("Requesting: " + url) resp = session.get(url, allow_redirects=False) if (resp.status_code) == 302: raise ValueError data = json.loads(resp.text) Debug.writeToFile( file_name, json.dumps(data, sort_keys=True, indent=4, separators=(",", ": ")), resp.encoding ) item = data["data"]["appInfo"] if not filter(lambda apk: apk.vercode == item["versionCode"], dAllApks[apkid]): v = item["version"].split(" ")[0] maxApkInfo = ApkVersionInfo(name=item["apkId"], ver=maxVerEachApk[item["apkId"]]) tmpApkInfo = ApkVersionInfo(name=item["apkId"], ver=v) # Is it >= maxVersion if maxApkInfo <= tmpApkInfo: thisSdk = int(item["sdkVersion"]) if thisSdk < minSdkEachApk[item["apkId"]]: logging.debug("SdkTooLow: {0}({1})".format(item["apkId"], thisSdk)) else: downloadApk( "http://download.mgccw.com/" + item["apkPath"], item["apkId"], item["version"], item["versionCode"], item["sdkVersion"], ) # END: if Sdk # END: if item except ValueError: logging.info("{0} not supported by mobogenie ...".format(apkid)) except: logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
def checkOneApp(apkid): """ checkOneApp(apkid): """ dAllApks = Global.dAllApks maxVerEachApk = Global.maxVerEachApk minSdkEachApk = Global.minSdkEachApk logging.info('Checking app: {0}'.format(apkid)) html_name = '{0}.html'.format(apkid) url = 'http://www.plazza.ir/app/' + apkid + '?hl=en' html = Debug.readFromFile(html_name) if html == '': session = requests.Session() session.proxies = Debug.getProxy() logging.debug('Requesting: ' + url) resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore') Debug.writeToFile(html_name, html, resp.encoding) try: dom = BeautifulSoup(html, 'html5lib') latestapk = dom.findAll('a', {'itemprop': 'downloadUrl'})[0] appid = re.search('(^\/dl\/)([0-9]+)(\/1$)', latestapk['href']).group(2) latesturl = session.head('http://www.plazza.ir' + latestapk['href'],allow_redirects=True).url latestver = re.search('(_)([0-9]+)(\.apk)$', latesturl).group(2) #We still miss versioncode comparison here downloadApk(latesturl,apkid,latestver) #Fetching of older versions is not completed, because it requires VIP accounts #olderapks = dom.findAll('div', {'style': 'direction: rtl'})[0].findAll('a', {'target': '_blank'}) #for apk in olderapks: # apkver = re.search('(\/)([0-9]+)(\?.*$|$)', apk['href']).group(2) #number is either end of string or there can be an ? for extra GET parameters # apkurl = session.head('http://www.plazza.ir/dl_version/' + appid + '/' + apkver + '/1',allow_redirects=True).url except AttributeError: logging.info('{0} has an invalid version in the download URL ...'.format(apkid)) except IndexError: logging.info('{0} not supported by plazza.ir ...'.format(apkid)) except: logging.exception('!!! Error parsing html from: "{0}"'.format(url))
def checkOneApp(self, apkid): """ checkOneApp(apkid): """ logging.info('Checking app: {0}'.format(apkid)) file_name = '{0}.json'.format(apkid) url = 'http://helper.mgccw.com/nclient/sjson/detail/detailInfo.htm?apkId=' + apkid data = Debug.readFromFile(file_name) try: if data == '': session = requests.Session() logging.debug('Requesting: ' + url) resp = session.get(url, allow_redirects=False) if (resp.status_code) == http.client.FOUND: raise ValueError data = json.loads(resp.text) Debug.writeToFile( file_name, json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')), resp.encoding) item = data['data']['appInfo'] avi = ApkVersionInfo(name=item['apkId'], sdk=item['sdkVersion'], ver=item['version'].split(' ')[0], vercode=item['versionCode'], download_src='http://download.mgccw.com/' + item['apkPath'], crawler_name=self.__class__.__name__) if self.report.isThisApkNeeded(avi): return self.downloadApk(avi) except ValueError: logging.info('{0} not supported by mobogenie ...'.format(apkid)) except: logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
def checkOneApp(apkid): """ checkOneApp(apkid): """ dAllApks = Global.dAllApks maxVerEachApk = Global.maxVerEachApk minSdkEachApk = Global.minSdkEachApk logging.info('Checking app: {0}'.format(apkid)) try: upToDownName = allUpToDownNames[apkid] html_name = '{0}.html'.format(upToDownName) url = 'http://' + upToDownName + '.en.uptodown.com/android/download' html = Debug.readFromFile(html_name) if html == '': session = requests.Session() session.proxies = Debug.getProxy() logging.debug('Requesting: ' + url) resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore') Debug.writeToFile(html_name, html, resp.encoding) try: dom = BeautifulSoup(html, 'html5lib') latestapk = dom.findAll('iframe', {'id': 'iframe_download'})[0]['src'] #note that this url will still result in a redirect 302 #We still miss versioncode comparison here downloadApk(latestapk,apkid) #We still miss fetching older versions except IndexError: logging.info('{0} not supported by uptodown.com ...'.format(apkid)) except: logging.exception('!!! Error parsing html from: "{0}"'.format(url)) except KeyError: logging.info('{0} not in uptodown.com dictionary...'.format(apkid))
def getUrlFromRedirect(apkname, url): """ getUrlFromRedirect(url): """ html_name = '{0}_redirect.html'.format(apkname) html = Debug.readFromFile(html_name) link = '' if html == '': session = requests.Session() session.proxies = Debug.getProxy() logging.debug('Requesting2: ' + url) resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore') Debug.writeToFile(html_name, html, resp.encoding) try: dom = BeautifulSoup(html, 'html5lib') link = dom.findAll('span', {'class': 'glyphicon glyphicon-cloud-download'})[0].parent['href'] except: logging.exception('!!! Error parsing html from: "{0}"'.format(url)) return link
def checkOneApp(self, apkid): """ checkOneApp(apkid): """ logging.info('Checking app: {0}'.format(apkid)) file_name = '{0}.json'.format(apkid) url = 'http://helper.mgccw.com/nclient/sjson/detail/detailInfo.htm?apkId=' + apkid data = Debug.readFromFile(file_name) try: if data == '': session = requests.Session() # session.proxies = Debug.getProxy() logging.debug('Requesting: ' + url) resp = session.get(url,allow_redirects=False) if (resp.status_code) == 302: raise ValueError data = json.loads(resp.text) Debug.writeToFile(file_name, json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')), resp.encoding) item=data['data']['appInfo'] avi = ApkVersionInfo(name=item['apkId'], sdk=item['sdkVersion'], ver=item['version'].split(' ')[0], vercode=item['versionCode'], download_src='http://download.mgccw.com/'+item['apkPath'] ) if self.report.isThisApkNeeded(avi): return self.downloadApk(avi) except ValueError: logging.info('{0} not supported by mobogenie ...'.format(apkid)) except: logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
def checkOneId(self, aptoideId): """ checkOneId(aptoideId): Get APK specific information from the Aptoide API and return it as an ApkVersionInfo object if it is tracked by OpenGApps """ file_name = '{0}.json'.format(aptoideId) url = 'http://ws75.aptoide.com/api/7/app/getMeta/app_id={0}'.format( aptoideId) data = Debug.readFromFile(file_name) run = {} run['id'] = aptoideId run['status'] = 'fail' # 'fail', 'empty', 'good' run['time'] = '' run['filename'] = '' if data == '': session = requests.Session() for x in range(1, 4): # up to three tries wait = GlobalDelay * x # logging.info('Waiting {0} seconds before fetching {1}'.format(wait, file_name)) time.sleep(wait) try: logging.debug('Checking ({0}): {1}'.format(x, url)) resp = session.get(url) if resp.status_code == http.client.OK: # Append ID on good http response run['status'] = 'empty' data = resp.json() if 'info' in data and 'status' in data[ 'info'] and data['info']['status'] == 'OK': # Found an APK update the Max. ID run['status'] = 'good' run['time'] = data['data']['modified'] theArchs = data['data']['file']['hardware'].get( 'cpus', []) _arch = 'all' if len(theArchs) > 0: _arch = ','.join(theArchs) avi = ApkVersionInfo( name=data['data']['package'], arch=_arch, sdk=data['data']['file']['hardware']['sdk'], dpi=self.doDpiStuff( data['data']['file']['hardware'].get( 'densities', [])), ver=data['data']['file']['vername'].split(' ') [0], # Look at only the true version number vercode=data['data']['file']['vercode'], download_src=data['data']['file']['path'], malware=( data['data']['file']['malware'] if 'malware' in data['data']['file'] else '' ), # We only have this key if vercode is in options crawler_name=self.__class__.__name__) Debug.writeToFile( file_name, json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')), resp.encoding) # Log AptoideID, Date, ApkID self.logIdAndDate(data['data']) # Check for beta support bCheckMore = False if self.report.needsBetaSupport(avi): import copy avibeta = copy.deepcopy(avi) avibeta.name += '.beta' needBeta = self.report.isThisApkNeeded(avibeta) # Do we already have it if self.report.isThisApkNeeded(avi): if ( avi.malware['rank'] == "warn" and avi.malware['reason'] ['signature_validated']['status'] == "failed" and avi.malware['reason'] ['signature_validated']['signature_from'] == "market" ): # signature matches market, but it does not pass verification logging.error( '{0} is a corrupt or incomplete APK, ignored.' .format(avi.download_src)) else: # Are we sure we still need it after the additional info? if self.report.isThisApkNeeded(avi): run['filename'] = self.downloadApk(avi) # END: if avi.malware if avi.name == 'org.opengapps.app': run['filename'] = '{0}-{1}_aptoideId-{2}.stub.apk'.format( avi.name, avi.vercode, aptoideId) else: pass # logging.error('data2[\'status\']: {0}, when fetching {1}, try {2}'.format(data.get('status', 'null'), file_name, x)) return run elif resp.status_code in [ http.client.UNAUTHORIZED, # 401 http.client.FORBIDDEN, # 403 http.client.NOT_FOUND, # 404 http.client.GONE ]: # 410 run['status'] = 'empty' return run else: pass # logging.error('HTTPStatus2: {0}, when fetching {1}, try {2}'.format(resp.status_code, file_name, x)) except: logging.exception( '!!! Invalid JSON from: "{0}", retry in: {1}s'.format( url, wait)) # END: for x # END: if data return run
def getAppVersions(apkInfo): """ getAppVersions(apkInfo): Collect all versions for an applicaiton """ logging.info('Fetching Information for: {0}'.format(apkInfo.apkmirror_name)) html_name = '{0}.html'.format(apkInfo.opengapps_name) url = APKMIRRORBASEURL + APKMIRRORGOOGLEURL2 + apkInfo.url html = Debug.readFromFile(html_name) if html == '': session = requests.Session() logging.debug('Requesting2: ' + url) resp = session.get(url) html = resp.text Debug.writeToFile(html_name, html, resp.encoding) try: dom = BeautifulSoup(html, 'html5lib') latest = dom.findAll('div', {'class': 'latestWidget'})[1] versions = latest.findAll('a', {'class': 'fontBlack'}) dVersions = {} for version in versions: # Ignore duplicate entries (one 'hidden' is shown, # and the other 'visible', is not shown) if 'visible-xs-block' in version.parent['class']: continue verText = '"{0}"'.format(version.get_text().encode('ascii', 'ignore')) if 'beta' in verText.lower() or 'preview' in verText.lower(): logging.info('!!! Beta or Preview Found: ' + verText) else: dVersions[verText] = version['href'] m = apkInfo.reVersion.search(verText) if m: avi = ApkVersionInfo(name=m.group('VERSIONNAME').rstrip('-.'), scrape_url=version['href']) avi.ver = avi.name avi.ver = avi.ver.replace(apkInfo.opengapps_name, '').strip() avi.ver = avi.ver.split(' ')[0] apkInfo.versions.append(avi) else: logging.info('!!! No Matchy: ' + verText) # END: for v in versions: Debug.printDictionary(dVersions) # Determine which versions to download if len(apkInfo.versions) > 0: maxVersionByName = sorted(apkInfo.versions)[-1] logging.debug('Max Version By Name: "{0}"'.format(maxVersionByName.name)) for v in apkInfo.versions: if v.name == maxVersionByName.name: logging.info('Getting Info for: "{0}" ({1})'.format(v.name, v.scrape_url)) getVersionInfo(v) logging.info('Downloading: "{0}"'.format(v.apk_name)) downloadApkFromVersionInfo(v) else: logging.debug('Skipping: "{0}" ({1})'.format(v.name, v.scrape_url)) # END: for v in apkInfo.versions: else: logging.info('No matching APKs found for: {0}'.format(apkInfo.apkmirror_name)) except: logging.exception('!!! Error parsing html from: "{0}"'.format(url)) logging.debug('-'*80)
def checkOneId(self, aptoideId): """ checkOneId(aptoideId): Get APK specific information from the Aptoide API and return it as an ApkVersionInfo object if it is tracked by OpenGApps """ file_name = '{0}.json'.format(aptoideId) url = 'http://webservices.aptoide.com/webservices/2/getApkInfo/id:{0}/json'.format(aptoideId) data = Debug.readFromFile(file_name) run = {} run['id'] = aptoideId run['status'] = 'fail' # 'fail', 'empty', 'good' run['time'] = '' run['filename'] = '' if data == '': session = requests.Session() for x in range(1, 4): # up to three tries wait = GlobalDelay * x # logging.info('Waiting {0} seconds before fetching {1}'.format(wait, file_name)) time.sleep(wait) try: logging.debug('Checking ({0}): {1}'.format(x, url)) resp = session.get(url) if resp.status_code == http.client.OK: # Append ID on good http response run['status'] = 'empty' data = resp.json() if 'status' in data and data['status'] == 'OK': # Found an APK update the Max. ID run['status'] = 'good' run['time'] = data['apk']['added'] avi = ApkVersionInfo(name =data['apk']['package'], arch =data['apk'].get('cpu', 'all'), sdk =data['apk']['minSdk'], dpi =self.doDpiStuff(data['apk'].get('screenCompat', 'nodpi')), ver =data['apk']['vername'].split(' ')[0], # Look at only the true version number vercode =data['apk']['vercode'], download_src=data['apk']['path'], malware=(data['malware'] if 'malware' in data else ''), # We only have this key if vercode is in options crawler_name=self.__class__.__name__ ) Debug.writeToFile(file_name, json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')), resp.encoding) # Log AptoideID, Date, ApkID self.logIdAndDate(data['apk']) # Check for beta support bCheckMore = False if self.report.needsBetaSupport(avi): import copy avibeta = copy.deepcopy(avi) avibeta.name += '.beta' needBeta = self.report.isThisApkNeeded(avibeta) # Do we already have it if self.report.isThisApkNeeded(avi): if (avi.malware['status'] == "warn" and avi.malware['reason']['signature_validated']['status'] == "failed" and avi.malware['reason']['signature_validated']['signature_from'] == "market"): # signature matches market, but it does not pass verification logging.error('{0} is a corrupt or incomplete APK, ignored.'.format(avi.download_src)) else: # Are we sure we still need it after the additional info? if self.report.isThisApkNeeded(avi): run['filename'] = self.downloadApk(avi) # END: if avi.malware if avi.name == 'org.opengapps.app': run['filename'] = '{0}-{1}_aptoideId-{2}.stub.apk'.format(avi.name, avi.vercode, aptoideId) else: pass # logging.error('data2[\'status\']: {0}, when fetching {1}, try {2}'.format(data.get('status', 'null'), file_name, x)) return run else: run['missingIds'] logging.error('HTTPStatus2: {0}, when fetching {1}, try {2}'.format(resp.status_code, file_name, x)) except: logging.exception('!!! Invalid JSON from: "{0}", retry in: {1}s'.format(url, wait)) # END: for x # END: if data return run
def checkOneApp(apkid): """ checkOneApp(apkid): """ dAllApks = Global.dAllApks maxVerEachApk = Global.maxVerEachApk minSdkEachApk = Global.minSdkEachApk logging.info('Checking app: {0}'.format(apkid)) html_name = '{0}.html'.format(apkid) url = 'http://apk-dl.com/' + apkid html = Debug.readFromFile(html_name) if html == '': session = requests.Session() session.proxies = Debug.getProxy() logging.debug('Requesting: ' + url) resp = session.get(url) html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore') Debug.writeToFile(html_name, html, resp.encoding) try: dom = BeautifulSoup(html, 'html5lib') apklist = dom.findAll('ul', {'class': 'apks dlist'})[0] apks = apklist.findAll('div', {'class': 'details'}) maxApkInfo = ApkVersionInfo(name=apkid, ver=maxVerEachApk[apkid]) for apk in apks: items = apk.findAll('div') dApk = {} for item in items: itext = '{0}'.format(item.get_text().encode('ascii', 'ignore')) itext = re.sub('\s', '', itext) itextsp = itext.split(':', 1) if len(itextsp) == 2: dApk[str(itextsp[0])] = str(itextsp[1]) dApk['url'] = 'http:' + apk.find('a', {'class': 'btn btn-success'})['href'] Debug.printDictionary(dApk) if 'Version' in dApk and 'RequiresAndroid' in dApk: (trash, sdk) = dApk['RequiresAndroid'].split('API:', 1) sdk = sdk[0:-1] (ver, vercode) = dApk['Version'].split('(Code:', 1) ver = ver.split('(', 1)[0] vercode = vercode[0:-1] tmpApkInfo = ApkVersionInfo(name=apkid, sdk=sdk, ver=ver, vercode=vercode) tmpApkInfo.download_url = dApk['url'] if maxApkInfo <= tmpApkInfo: thisSdk = int(tmpApkInfo.sdk) if thisSdk < minSdkEachApk[apkid]: logging.debug('SdkTooLow: {0}({1})'.format(apkid, thisSdk)) continue if not filter(lambda apk: apk.vercode == tmpApkInfo.vercode, dAllApks[apkid]): logging.debug(tmpApkInfo.fullString(maxVerEachApk[apkid])) downloadApk(tmpApkInfo) except IndexError: logging.info('{0} not supported by apk-dl.com ...'.format(apkid)) except: logging.exception('!!! Error parsing html from: "{0}"'.format(url))