def listRepo(repo, orderby=None):
"""
listRepo(repo): Get list of all APKs in a specific store from the Aptoide API
and return it in JSON form
"""
file_name = '{0}{1}.json'.format(repo, '' if not orderby else '-' + '-'.join(orderby))
orderby = '' if not orderby else '/orderby/' + '/'.join(orderby)
url = 'http://webservices.aptoide.com/webservices/listRepository/{0}{1}/json'.format(repo, orderby)
data = Debug.readFromFile(file_name)
try:
if data == '':
session = requests.Session()
# session.proxies = Debug.getProxy()
logging.debug('Requesting1: ' + url)
resp = session.get(url)
data = json.loads(resp.text)
Debug.writeToFile(file_name, json.dumps(data, sort_keys=True,
indent=4, separators=(',', ': ')), resp.encoding)
if data['status'] == 'OK':
return data
else:
logging.error(file_name)
except:
logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
return None
def getVersionInfo(apkVersionInfo):
"""
getVersionInfo(apkVersionInfo): Determines each versions information
"""
html_name = apkVersionInfo.scrape_url.rsplit('/', 2)[1]
html_name = html_name.replace('-android-apk-download', '') + '.html'
url = APKMIRRORBASEURL + apkVersionInfo.scrape_url
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
logging.debug('Requesting3: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
postArea = dom.findAll('div', {'class': 'post-area'})[0]
dl_button = postArea.findAll('a', {'type': 'button'})[1]
blueFonts = postArea.findAll('span', {'class': 'fontBlue'})
apkVersionInfo.download_url = dl_button['href']
for blueFont in blueFonts:
if blueFont.get_text() == 'File name: ':
apkVersionInfo.apk_name = blueFont.next_sibling
if blueFont.get_text() == 'Version: ':
apkVersionInfo.ver = blueFont.next_sibling
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
def getApkInfo(repo, apkid, apkversion, options=None, doVersion1=False):
"""
getApkInfo(repo, apkid, apkversion): Get APK specific information from the Aptoide API
and return it in JSON form
"""
version = '1' if doVersion1 else '2'
file_name = '{0}-{1}-{2}_{3}.json'.format(repo, apkid, apkversion, version)
options = '' if not options else '/options=({0})'.format(options)
url = 'http://webservices.aptoide.com/webservices/{0}/getApkInfo/{1}/{2}/{3}{4}/json'.format(
version, repo, apkid, apkversion, options)
data = Debug.readFromFile(file_name)
try:
if data == '':
session = requests.Session()
# session.proxies = Debug.getProxy()
logging.debug('Requesting2: ' + url)
resp = session.get(url)
data = json.loads(resp.text)
Debug.writeToFile(file_name, json.dumps(data, sort_keys=True,
indent=4, separators=(',', ': ')), resp.encoding)
if data['status'] == 'OK':
return data
else:
logging.error(file_name)
except:
logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
return None
def checkOneApp(apkid):
"""
checkOneApp(apkid):
"""
dAllApks = Global.dAllApks
maxVerEachApk = Global.maxVerEachApk
minSdkEachApk = Global.minSdkEachApk
logging.info("Checking app: {0}".format(apkid))
file_name = "{0}.json".format(apkid)
url = "http://helper.mgccw.com/nclient/sjson/detail/detailInfo.htm?apkId=" + apkid
data = Debug.readFromFile(file_name)
try:
if data == "":
session = requests.Session()
# session.proxies = Debug.getProxy()
logging.debug("Requesting: " + url)
resp = session.get(url, allow_redirects=False)
if (resp.status_code) == 302:
raise ValueError
data = json.loads(resp.text)
Debug.writeToFile(
file_name, json.dumps(data, sort_keys=True, indent=4, separators=(",", ": ")), resp.encoding
)
item = data["data"]["appInfo"]
if not filter(lambda apk: apk.vercode == item["versionCode"], dAllApks[apkid]):
v = item["version"].split(" ")[0]
maxApkInfo = ApkVersionInfo(name=item["apkId"], ver=maxVerEachApk[item["apkId"]])
tmpApkInfo = ApkVersionInfo(name=item["apkId"], ver=v)
# Is it >= maxVersion
if maxApkInfo <= tmpApkInfo:
thisSdk = int(item["sdkVersion"])
if thisSdk < minSdkEachApk[item["apkId"]]:
logging.debug("SdkTooLow: {0}({1})".format(item["apkId"], thisSdk))
else:
downloadApk(
"http://download.mgccw.com/" + item["apkPath"],
item["apkId"],
item["version"],
item["versionCode"],
item["sdkVersion"],
)
# END: if Sdk
# END: if item
except ValueError:
logging.info("{0} not supported by mobogenie ...".format(apkid))
except:
logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
def checkOneApp(apkid):
"""
checkOneApp(apkid):
"""
dAllApks = Global.dAllApks
maxVerEachApk = Global.maxVerEachApk
minSdkEachApk = Global.minSdkEachApk
logging.info('Checking app: {0}'.format(apkid))
html_name = '{0}.html'.format(apkid)
url = 'http://www.plazza.ir/app/' + apkid + '?hl=en'
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
session.proxies = Debug.getProxy()
logging.debug('Requesting: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
latestapk = dom.findAll('a', {'itemprop': 'downloadUrl'})[0]
appid = re.search('(^\/dl\/)([0-9]+)(\/1$)', latestapk['href']).group(2)
latesturl = session.head('http://www.plazza.ir' + latestapk['href'],allow_redirects=True).url
latestver = re.search('(_)([0-9]+)(\.apk)$', latesturl).group(2)
#We still miss versioncode comparison here
downloadApk(latesturl,apkid,latestver)
#Fetching of older versions is not completed, because it requires VIP accounts
#olderapks = dom.findAll('div', {'style': 'direction: rtl'})[0].findAll('a', {'target': '_blank'})
#for apk in olderapks:
# apkver = re.search('(\/)([0-9]+)(\?.*$|$)', apk['href']).group(2) #number is either end of string or there can be an ? for extra GET parameters
# apkurl = session.head('http://www.plazza.ir/dl_version/' + appid + '/' + apkver + '/1',allow_redirects=True).url
except AttributeError:
logging.info('{0} has an invalid version in the download URL ...'.format(apkid))
except IndexError:
logging.info('{0} not supported by plazza.ir ...'.format(apkid))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
def checkOneApp(self, apkid):
"""
checkOneApp(apkid):
"""
logging.info('Checking app: {0}'.format(apkid))
file_name = '{0}.json'.format(apkid)
url = 'http://helper.mgccw.com/nclient/sjson/detail/detailInfo.htm?apkId=' + apkid
data = Debug.readFromFile(file_name)
try:
if data == '':
session = requests.Session()
logging.debug('Requesting: ' + url)
resp = session.get(url, allow_redirects=False)
if (resp.status_code) == http.client.FOUND:
raise ValueError
data = json.loads(resp.text)
Debug.writeToFile(
file_name,
json.dumps(data,
sort_keys=True,
indent=4,
separators=(',', ': ')), resp.encoding)
item = data['data']['appInfo']
avi = ApkVersionInfo(name=item['apkId'],
sdk=item['sdkVersion'],
ver=item['version'].split(' ')[0],
vercode=item['versionCode'],
download_src='http://download.mgccw.com/' +
item['apkPath'],
crawler_name=self.__class__.__name__)
if self.report.isThisApkNeeded(avi):
return self.downloadApk(avi)
except ValueError:
logging.info('{0} not supported by mobogenie ...'.format(apkid))
except:
logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
def checkOneApp(apkid):
"""
checkOneApp(apkid):
"""
dAllApks = Global.dAllApks
maxVerEachApk = Global.maxVerEachApk
minSdkEachApk = Global.minSdkEachApk
logging.info('Checking app: {0}'.format(apkid))
try:
upToDownName = allUpToDownNames[apkid]
html_name = '{0}.html'.format(upToDownName)
url = 'http://' + upToDownName + '.en.uptodown.com/android/download'
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
session.proxies = Debug.getProxy()
logging.debug('Requesting: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
latestapk = dom.findAll('iframe', {'id': 'iframe_download'})[0]['src'] #note that this url will still result in a redirect 302
#We still miss versioncode comparison here
downloadApk(latestapk,apkid)
#We still miss fetching older versions
except IndexError:
logging.info('{0} not supported by uptodown.com ...'.format(apkid))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
except KeyError:
logging.info('{0} not in uptodown.com dictionary...'.format(apkid))
def getUrlFromRedirect(apkname, url):
"""
getUrlFromRedirect(url):
"""
html_name = '{0}_redirect.html'.format(apkname)
html = Debug.readFromFile(html_name)
link = ''
if html == '':
session = requests.Session()
session.proxies = Debug.getProxy()
logging.debug('Requesting2: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
link = dom.findAll('span', {'class': 'glyphicon glyphicon-cloud-download'})[0].parent['href']
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
return link
def checkOneApp(self, apkid):
"""
checkOneApp(apkid):
"""
logging.info('Checking app: {0}'.format(apkid))
file_name = '{0}.json'.format(apkid)
url = 'http://helper.mgccw.com/nclient/sjson/detail/detailInfo.htm?apkId=' + apkid
data = Debug.readFromFile(file_name)
try:
if data == '':
session = requests.Session()
# session.proxies = Debug.getProxy()
logging.debug('Requesting: ' + url)
resp = session.get(url,allow_redirects=False)
if (resp.status_code) == 302:
raise ValueError
data = json.loads(resp.text)
Debug.writeToFile(file_name, json.dumps(data, sort_keys=True,
indent=4, separators=(',', ': ')), resp.encoding)
item=data['data']['appInfo']
avi = ApkVersionInfo(name=item['apkId'],
sdk=item['sdkVersion'],
ver=item['version'].split(' ')[0],
vercode=item['versionCode'],
download_src='http://download.mgccw.com/'+item['apkPath']
)
if self.report.isThisApkNeeded(avi):
return self.downloadApk(avi)
except ValueError:
logging.info('{0} not supported by mobogenie ...'.format(apkid))
except:
logging.exception('!!! Invalid JSON from: "{0}"'.format(url))
def checkOneId(self, aptoideId):
"""
checkOneId(aptoideId): Get APK specific information from the Aptoide API
and return it as an ApkVersionInfo object if it is
tracked by OpenGApps
"""
file_name = '{0}.json'.format(aptoideId)
url = 'http://ws75.aptoide.com/api/7/app/getMeta/app_id={0}'.format(
aptoideId)
data = Debug.readFromFile(file_name)
run = {}
run['id'] = aptoideId
run['status'] = 'fail' # 'fail', 'empty', 'good'
run['time'] = ''
run['filename'] = ''
if data == '':
session = requests.Session()
for x in range(1, 4): # up to three tries
wait = GlobalDelay * x
# logging.info('Waiting {0} seconds before fetching {1}'.format(wait, file_name))
time.sleep(wait)
try:
logging.debug('Checking ({0}): {1}'.format(x, url))
resp = session.get(url)
if resp.status_code == http.client.OK:
# Append ID on good http response
run['status'] = 'empty'
data = resp.json()
if 'info' in data and 'status' in data[
'info'] and data['info']['status'] == 'OK':
# Found an APK update the Max. ID
run['status'] = 'good'
run['time'] = data['data']['modified']
theArchs = data['data']['file']['hardware'].get(
'cpus', [])
_arch = 'all'
if len(theArchs) > 0:
_arch = ','.join(theArchs)
avi = ApkVersionInfo(
name=data['data']['package'],
arch=_arch,
sdk=data['data']['file']['hardware']['sdk'],
dpi=self.doDpiStuff(
data['data']['file']['hardware'].get(
'densities', [])),
ver=data['data']['file']['vername'].split(' ')
[0], # Look at only the true version number
vercode=data['data']['file']['vercode'],
download_src=data['data']['file']['path'],
malware=(
data['data']['file']['malware'] if
'malware' in data['data']['file'] else ''
), # We only have this key if vercode is in options
crawler_name=self.__class__.__name__)
Debug.writeToFile(
file_name,
json.dumps(data,
sort_keys=True,
indent=4,
separators=(',', ': ')),
resp.encoding)
# Log AptoideID, Date, ApkID
self.logIdAndDate(data['data'])
# Check for beta support
bCheckMore = False
if self.report.needsBetaSupport(avi):
import copy
avibeta = copy.deepcopy(avi)
avibeta.name += '.beta'
needBeta = self.report.isThisApkNeeded(avibeta)
# Do we already have it
if self.report.isThisApkNeeded(avi):
if (
avi.malware['rank'] == "warn"
and avi.malware['reason']
['signature_validated']['status']
== "failed" and avi.malware['reason']
['signature_validated']['signature_from']
== "market"
): # signature matches market, but it does not pass verification
logging.error(
'{0} is a corrupt or incomplete APK, ignored.'
.format(avi.download_src))
else:
# Are we sure we still need it after the additional info?
if self.report.isThisApkNeeded(avi):
run['filename'] = self.downloadApk(avi)
# END: if avi.malware
if avi.name == 'org.opengapps.app':
run['filename'] = '{0}-{1}_aptoideId-{2}.stub.apk'.format(
avi.name, avi.vercode, aptoideId)
else:
pass # logging.error('data2[\'status\']: {0}, when fetching {1}, try {2}'.format(data.get('status', 'null'), file_name, x))
return run
elif resp.status_code in [
http.client.UNAUTHORIZED, # 401
http.client.FORBIDDEN, # 403
http.client.NOT_FOUND, # 404
http.client.GONE
]: # 410
run['status'] = 'empty'
return run
else:
pass # logging.error('HTTPStatus2: {0}, when fetching {1}, try {2}'.format(resp.status_code, file_name, x))
except:
logging.exception(
'!!! Invalid JSON from: "{0}", retry in: {1}s'.format(
url, wait))
# END: for x
# END: if data
return run
def getAppVersions(apkInfo):
"""
getAppVersions(apkInfo): Collect all versions for an applicaiton
"""
logging.info('Fetching Information for: {0}'.format(apkInfo.apkmirror_name))
html_name = '{0}.html'.format(apkInfo.opengapps_name)
url = APKMIRRORBASEURL + APKMIRRORGOOGLEURL2 + apkInfo.url
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
logging.debug('Requesting2: ' + url)
resp = session.get(url)
html = resp.text
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
latest = dom.findAll('div', {'class': 'latestWidget'})[1]
versions = latest.findAll('a', {'class': 'fontBlack'})
dVersions = {}
for version in versions:
# Ignore duplicate entries (one 'hidden' is shown,
# and the other 'visible', is not shown)
if 'visible-xs-block' in version.parent['class']:
continue
verText = '"{0}"'.format(version.get_text().encode('ascii', 'ignore'))
if 'beta' in verText.lower() or 'preview' in verText.lower():
logging.info('!!! Beta or Preview Found: ' + verText)
else:
dVersions[verText] = version['href']
m = apkInfo.reVersion.search(verText)
if m:
avi = ApkVersionInfo(name=m.group('VERSIONNAME').rstrip('-.'), scrape_url=version['href'])
avi.ver = avi.name
avi.ver = avi.ver.replace(apkInfo.opengapps_name, '').strip()
avi.ver = avi.ver.split(' ')[0]
apkInfo.versions.append(avi)
else:
logging.info('!!! No Matchy: ' + verText)
# END: for v in versions:
Debug.printDictionary(dVersions)
# Determine which versions to download
if len(apkInfo.versions) > 0:
maxVersionByName = sorted(apkInfo.versions)[-1]
logging.debug('Max Version By Name: "{0}"'.format(maxVersionByName.name))
for v in apkInfo.versions:
if v.name == maxVersionByName.name:
logging.info('Getting Info for: "{0}" ({1})'.format(v.name, v.scrape_url))
getVersionInfo(v)
logging.info('Downloading: "{0}"'.format(v.apk_name))
downloadApkFromVersionInfo(v)
else:
logging.debug('Skipping: "{0}" ({1})'.format(v.name, v.scrape_url))
# END: for v in apkInfo.versions:
else:
logging.info('No matching APKs found for: {0}'.format(apkInfo.apkmirror_name))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))
logging.debug('-'*80)
def checkOneId(self, aptoideId):
"""
checkOneId(aptoideId): Get APK specific information from the Aptoide API
and return it as an ApkVersionInfo object if it is
tracked by OpenGApps
"""
file_name = '{0}.json'.format(aptoideId)
url = 'http://webservices.aptoide.com/webservices/2/getApkInfo/id:{0}/json'.format(aptoideId)
data = Debug.readFromFile(file_name)
run = {}
run['id'] = aptoideId
run['status'] = 'fail' # 'fail', 'empty', 'good'
run['time'] = ''
run['filename'] = ''
if data == '':
session = requests.Session()
for x in range(1, 4): # up to three tries
wait = GlobalDelay * x
# logging.info('Waiting {0} seconds before fetching {1}'.format(wait, file_name))
time.sleep(wait)
try:
logging.debug('Checking ({0}): {1}'.format(x, url))
resp = session.get(url)
if resp.status_code == http.client.OK:
# Append ID on good http response
run['status'] = 'empty'
data = resp.json()
if 'status' in data and data['status'] == 'OK':
# Found an APK update the Max. ID
run['status'] = 'good'
run['time'] = data['apk']['added']
avi = ApkVersionInfo(name =data['apk']['package'],
arch =data['apk'].get('cpu', 'all'),
sdk =data['apk']['minSdk'],
dpi =self.doDpiStuff(data['apk'].get('screenCompat', 'nodpi')),
ver =data['apk']['vername'].split(' ')[0], # Look at only the true version number
vercode =data['apk']['vercode'],
download_src=data['apk']['path'],
malware=(data['malware'] if 'malware' in data else ''), # We only have this key if vercode is in options
crawler_name=self.__class__.__name__
)
Debug.writeToFile(file_name, json.dumps(data, sort_keys=True,
indent=4, separators=(',', ': ')), resp.encoding)
# Log AptoideID, Date, ApkID
self.logIdAndDate(data['apk'])
# Check for beta support
bCheckMore = False
if self.report.needsBetaSupport(avi):
import copy
avibeta = copy.deepcopy(avi)
avibeta.name += '.beta'
needBeta = self.report.isThisApkNeeded(avibeta)
# Do we already have it
if self.report.isThisApkNeeded(avi):
if (avi.malware['status'] == "warn" and
avi.malware['reason']['signature_validated']['status'] == "failed" and
avi.malware['reason']['signature_validated']['signature_from'] == "market"): # signature matches market, but it does not pass verification
logging.error('{0} is a corrupt or incomplete APK, ignored.'.format(avi.download_src))
else:
# Are we sure we still need it after the additional info?
if self.report.isThisApkNeeded(avi):
run['filename'] = self.downloadApk(avi)
# END: if avi.malware
if avi.name == 'org.opengapps.app':
run['filename'] = '{0}-{1}_aptoideId-{2}.stub.apk'.format(avi.name, avi.vercode, aptoideId)
else:
pass # logging.error('data2[\'status\']: {0}, when fetching {1}, try {2}'.format(data.get('status', 'null'), file_name, x))
return run
else:
run['missingIds']
logging.error('HTTPStatus2: {0}, when fetching {1}, try {2}'.format(resp.status_code, file_name, x))
except:
logging.exception('!!! Invalid JSON from: "{0}", retry in: {1}s'.format(url, wait))
# END: for x
# END: if data
return run
def checkOneApp(apkid):
"""
checkOneApp(apkid):
"""
dAllApks = Global.dAllApks
maxVerEachApk = Global.maxVerEachApk
minSdkEachApk = Global.minSdkEachApk
logging.info('Checking app: {0}'.format(apkid))
html_name = '{0}.html'.format(apkid)
url = 'http://apk-dl.com/' + apkid
html = Debug.readFromFile(html_name)
if html == '':
session = requests.Session()
session.proxies = Debug.getProxy()
logging.debug('Requesting: ' + url)
resp = session.get(url)
html = unicodedata.normalize('NFKD', resp.text).encode('ascii', 'ignore')
Debug.writeToFile(html_name, html, resp.encoding)
try:
dom = BeautifulSoup(html, 'html5lib')
apklist = dom.findAll('ul', {'class': 'apks dlist'})[0]
apks = apklist.findAll('div', {'class': 'details'})
maxApkInfo = ApkVersionInfo(name=apkid, ver=maxVerEachApk[apkid])
for apk in apks:
items = apk.findAll('div')
dApk = {}
for item in items:
itext = '{0}'.format(item.get_text().encode('ascii', 'ignore'))
itext = re.sub('\s', '', itext)
itextsp = itext.split(':', 1)
if len(itextsp) == 2:
dApk[str(itextsp[0])] = str(itextsp[1])
dApk['url'] = 'http:' + apk.find('a', {'class': 'btn btn-success'})['href']
Debug.printDictionary(dApk)
if 'Version' in dApk and 'RequiresAndroid' in dApk:
(trash, sdk) = dApk['RequiresAndroid'].split('API:', 1)
sdk = sdk[0:-1]
(ver, vercode) = dApk['Version'].split('(Code:', 1)
ver = ver.split('(', 1)[0]
vercode = vercode[0:-1]
tmpApkInfo = ApkVersionInfo(name=apkid, sdk=sdk, ver=ver, vercode=vercode)
tmpApkInfo.download_url = dApk['url']
if maxApkInfo <= tmpApkInfo:
thisSdk = int(tmpApkInfo.sdk)
if thisSdk < minSdkEachApk[apkid]:
logging.debug('SdkTooLow: {0}({1})'.format(apkid, thisSdk))
continue
if not filter(lambda apk: apk.vercode == tmpApkInfo.vercode, dAllApks[apkid]):
logging.debug(tmpApkInfo.fullString(maxVerEachApk[apkid]))
downloadApk(tmpApkInfo)
except IndexError:
logging.info('{0} not supported by apk-dl.com ...'.format(apkid))
except:
logging.exception('!!! Error parsing html from: "{0}"'.format(url))