def run(packagename):
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
# Connect
api = GooglePlayAPI(config.ANDROID_ID)
api.login(config.GOOGLE_LOGIN, config.GOOGLE_PASSWORD, config.AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def download(self, packageName):
"""Download an app and return its raw data (APK file).
packageName is the app unique ID (usually starting with 'com.').
versionCode can be grabbed by using the details() method on the given
app."""
# Get the version code and the offer type from the app details
doc = self.details(packageName).docV2
versionCode = doc.details.appDetails.versionCode
offerType = doc.offer[0].offerType
path = "purchase"
data = "ot=%d&doc=%s&vc=%d" % (offerType, packageName, versionCode)
message = self.executeRequestApi2(path, data)
url = message.payload.buyResponse.purchaseStatusResponse.appDeliveryData.downloadUrl
cookie = message.payload.buyResponse.purchaseStatusResponse.appDeliveryData.downloadAuthCookie[0]
cookies = {
str(cookie.name): str(cookie.value) # python-requests #459 fixes this
}
headers = {
"User-Agent" : "AndroidDownloadManager/4.1.1 (Linux; U; Android 4.1.1; Nexus S Build/JRO03E)",
"Accept-Encoding": "",
}
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
response = requests.get(url, headers=headers, cookies=cookies, verify=False)
return response.content
def getDetailsForDownload(api, app):
m = api.details(app)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
sz = sizeof_fmt(doc.details.appDetails.installationSize)
return vc, ot, sz
def getDetailsForDownload(api, app):
m = api.details(app)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
sz = sizeof_fmt(doc.details.appDetails.installationSize)
return vc, ot, sz
def plot_speed_efficiency(df,
filesize,
title=True,
legend=True,
yaxis=True,
xaxis=True):
"""
Given a pandas Dataframe, plots each protocol's speed
vs it's data efficiency
"""
rows = df.loc[int(filesize)]
x = rows["Bytes Total"]
y = rows["Speed (bytes/s)"]
for i in range(len(rows)):
pname = rows.index[i]
plt.scatter(x[i],
y[i],
marker=MARKERS[i],
color=COLORS[i],
s=150,
label=pname)
# Add a line to represent filesize
ylim = plt.ylim()
plt.plot([filesize, filesize], [ylim[0], ylim[1]],
"r--",
linewidth=3,
label="filesize")
# Add surrounding information
if yaxis:
plt.ylabel("Speed (bytes/s)", size=24)
if xaxis:
plt.xlabel("Total Bytes Transferred", size=24)
if title:
plt.title("Speed vs Data Efficiency for %s File" %
sizeof_fmt(filesize),
size=26,
y=1.1)
if legend:
plt.legend(bbox_to_anchor=(1.3, 1),
scatterpoints=1,
prop={"size": "20"})
# Make sure the limits are correct and consistent
plt.axes().yaxis.set_major_locator(plt.MaxNLocator(4))
plt.axes().xaxis.set_major_locator(plt.MaxNLocator(4))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
plt.ylim(ylim)
plt.show()
def plot_packets(batch_id):
"""
Plots time on the x axis vs length on the y axis
"""
dump_batch_names = get_fnames_by_id(batch_id)
if len(dump_batch_names) < len(PROC_ARGS):
print "At least one protocol was missing from batch %s" % batch_id
return
dump_protocols = []
filesize = 0
# Convert all the times to an offset from 0
xy = []
for name in sorted(dump_batch_names):
with open("packet_dumps/%s" % name) as f:
dump = json.load(f)
# If any of the dumps don't have a packets array, abort the whole dump
if not dump.get("stored_packets", True):
print "Dump %s was not captured with --store-packets" % name
return
dump_protocols.append(dump["protocol"])
filesize = sizeof_fmt(dump["file_size"])
x = [datetime.strptime(l["time"], TIME_FORMAT) for l in dump["packets"]]
t0 = min(x)
dt = timedelta(hours=t0.hour, minutes=t0.minute, seconds=t0.second,
microseconds=t0.microsecond)
x = [t-dt for t in x]
y = [l["length"] for l in dump["packets"]]
xy.append((x, y))
fig, axes = plt.subplots(len(PROC_ARGS), sharex="col")
max_yticks = 4
for i in range(len(PROC_ARGS)):
axes[i].plot(xy[i][0], xy[i][1], ".")
axes[i].xaxis.set_major_formatter(dates.DateFormatter(PLOT_TIME_FORMAT))
axes[i].set_title(dump_protocols[i], size=16)
axes[i].yaxis.set_major_locator(plt.MaxNLocator(max_yticks))
# Add a line to show where the transfer stopped
ylim = axes[i].get_ylim()
end = max(xy[i][0])
axes[i].plot([end, end], [ylim[0], ylim[1]], "g--", linewidth=2, label="final packet")
fig.text(0.5, 0.04, "Time elapsed", ha='center', va='center', size=16)
fig.text(0.04, 0.5, "Length of payload (bytes)", ha='center', va='center',
rotation='vertical', size=16)
fig.suptitle("Individual Packets for %s Transfer" % filesize, size=18)
axes[len(PROC_ARGS)-1].legend(bbox_to_anchor=(1, -0.1))
fig.show()
def scrape(id, package):
packagename = package
res = urllib2.urlopen("https://play.google.com/store/apps/details?id=%s&hl=en" % packagename)
html = res.read()
path = "assets/%d/" % id
if not os.path.exists(path):
os.makedirs(path)
name = html.split('itemprop="name"')[1].split("</div>")[0].split("<div>")[1]
desc = html.split('<div class="show-more-content text-body" itemprop="description">')[1].split(
'<div class="show-more-end">'
)[0]
rating = html.split("Rated ")[1].split(" stars")[0]
category = html.split('<span itemprop="genre">')[1].split("</span>")[0]
stuff = []
for line in html.split("</div>"):
if "full-screenshot" in line:
url = line.split('src="')[1][:-3]
stuff.append(url)
x = 0
for img in stuff[1:]:
print img
urllib.urlretrieve(img, "%s%d.webp" % (path, x))
x += 1
filename = path + packagename + ".apk"
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
ap = apk.APK(filename)
badging = os.popen("/home/thomas/dev/android-odroid/out/host/linux-x86/bin/aapt dump badging %s" % filename).read()
for line in badging.split("\n"):
if "launchable-activity" in line:
activity = line.split("'")[1]
return [name, desc, rating, package, activity, category]
def downloadApps(api,packagename,filename):
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def downloadApps(api, packagename, filename):
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def download_apk(packagename, filename):
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def downloadApps(api,packagename,filename):
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
if Desc==True:
getDescriptions(doc)
print
if Perm==True:
getPermissions(doc)
#print doc.details.appDetails.permission
# Download
if Down==True:
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def get_apk(packagename, friendly_name): # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType easy_name = friendly_name + "-" + str(vc) filename = easy_name +".apk" # logic to check if we have this version here if do_we_already_have(friendly_name, easy_name): print "We already have " + easy_name + " skipping download" else: print "Downloading " + easy_name + " from the Google Play Store." # Download print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def plot_speed_efficiency(df, filesize, title=True, legend=True, yaxis=True, xaxis=True):
"""
Given a pandas Dataframe, plots each protocol's speed
vs it's data efficiency
"""
rows = df.loc[int(filesize)]
x = rows["Bytes Total"]
y = rows["Speed (bytes/s)"]
for i in range(len(rows)):
pname = rows.index[i]
plt.scatter(x[i], y[i], marker=MARKERS[i], color=COLORS[i], s=150, label=pname)
# Add a line to represent filesize
ylim = plt.ylim()
plt.plot([filesize, filesize], [ylim[0], ylim[1]], "r--", linewidth=3, label="filesize")
# Add surrounding information
if yaxis:
plt.ylabel("Speed (bytes/s)", size=24)
if xaxis:
plt.xlabel("Total Bytes Transferred", size=24)
if title:
plt.title("Speed vs Data Efficiency for %s File" % sizeof_fmt(filesize), size=26, y=1.1)
if legend:
plt.legend(bbox_to_anchor=(1.3, 1), scatterpoints=1, prop={"size":"20"})
# Make sure the limits are correct and consistent
plt.axes().yaxis.set_major_locator(plt.MaxNLocator(4))
plt.axes().xaxis.set_major_locator(plt.MaxNLocator(4))
plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
plt.ylim(ylim)
plt.show()
if ((len(sys.argv) >= 3) and (len(sys.argv)<5)):
filename = sys.argv[2]
google_pass = sys.argv[3]
elif (len(sys.argv)<3):
filename = packagename + ".apk"
google_pass = getpass.getpass()
else:
print "Usage: %s packagename filename, password]"
print "Download an app."
print "If filename is not present, will write to packagename.apk."
print "If password is not present, user will be prompted for one"
sys.exit(0)
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, google_pass, AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def downloadApkAndUpdateDB(api, db, packagename, fileDir):
filename = fileDir + '/' + packagename + ".apk"
try:
m = api.details(packagename)
except:
sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename))
traceback.print_exc(file=sys.stderr)
return (False, traceback.format_exc())
doc = m.docV2
vc = doc.details.appDetails.versionCode
# Zero length offer means the app is not found, possibly removed from the
# play store.
# We treat this as a success case so we can treat this app as processed
# and avoid re-downloading it if we need to recover from an exception.
if (len(doc.offer) == 0):
sys.stdout.write("%d \t %s \t Not Found\n" %
(int(time.time()), packagename))
return (True, packagename + " Not Found")
try:
ot = doc.offer[0].offerType
except:
sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename))
print >> sys.stderr, doc
traceback.print_exc(file=sys.stderr)
return (False, traceback.format_exc())
packageName = doc.details.appDetails.packageName
#use eval, since only with api.toDict, pymongo will throw some warning related to wsgi
docDict = eval(str(api.toDict(doc)))
isApkUpdated = False
#apkDetails is collection for doc + updated timestamp
#if docDict does change, update the db entry
#use more query instead of insertion to speed up
#Do not get fields not in docDict from apkDetails
preDetailsEntry = db.apkDetails.find_one(
{'details.appDetails.packageName': packageName}, {
'updatedTimestamp': 0,
'_id': 0
})
if preDetailsEntry != docDict:
#Warning: sometimes versionCode is not available
#versionCode is used for determine whether apk has been updated
#http://developer.android.com/tools/publishing/versioning.html
try:
isApkUpdated = (not preDetailsEntry) or (
docDict['details']['appDetails']['versionCode'] !=
preDetailsEntry['details']['appDetails']['versionCode'])
except KeyError as e:
isApkUpdated = True
docDict['updatedTimestamp'] = datetime.datetime.utcnow()
db.apkDetails.update({'details.appDetails.packageName': packageName},
docDict,
upsert=True)
else:
isApkUpdated = False
infoDict = docDict['details']['appDetails']
isFree = not doc.offer[0].checkoutFlowRequired
isCurrentVersionDownloaded = False
# 2015-04-17: Removed apk size restriction.
# According to
# http://android-developers.blogspot.com/2012/03/android-apps-break-50mb-barrier.html
# Each apk file will be at most 50 mb. An app that is listed as bigger than
# 50 mb on the playstore will have expansion packs that can supply
# additional things like graphics, etc.
# http://developer.android.com/distribute/googleplay/publish/preparing.html#size
isSizeExceed = False
# if doc.details.appDetails.installationSize > 52428800:
# isSizeExceed = True
# else:
# isSizeExceed = False
#Remove db entry fields which are not in infoDict
preInfoEntry = db.apkInfo.find_one({'packageName': packageName}, {
'isFree': 0,
'isSizeExceed': 0,
'updatedTimestamp': 0,
'_id': 0
})
if preInfoEntry == None:
preInfoEntry = {}
preIsDownloaded = preInfoEntry.pop('isDownloaded', False)
preIsCurrentVersionDownloaded = preInfoEntry.pop(
'isCurrentVersionDownloaded', False)
preIsApkUpdated = preInfoEntry.pop('isApkUpdated', False)
preFileDir = preInfoEntry.pop('fileDir', '')
# Default return value
ret = (True, packageName)
# Download when it is free and not exceed 50 mb and (current version in apkInfo was not downloaded or app has been updated since last time update apkInfo version)
if isFree and (not isSizeExceed) and ((not preIsCurrentVersionDownloaded)
or isApkUpdated):
try:
data = api.download(packageName, vc, ot)
except Exception as e:
sys.stderr.write("%d \t %s\n" % (int(time.time()), packageName))
traceback.print_exc(file=sys.stderr)
isCurrentVersionDownloaded = False
ret = (False, traceback.format_exc())
else:
print "%d \t %s" % (int(time.time()), packageName)
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
if preFileDir != '':
fileDir = preFileDir
filename = preFileDir + '/' + packagename + ".apk"
open(filename, "wb").write(data)
print "Done"
isCurrentVersionDownloaded = True
else:
if isApkUpdated == False and preIsCurrentVersionDownloaded == True:
isCurrentVersionDownloaded = True
s = "Escape downloading isFree: %s, isSizeExceed: %s, preIsCurrentVersionDownloaded: %s, isApkUpdated: %s" % (
isFree, isSizeExceed, preIsCurrentVersionDownloaded, isApkUpdated)
print "%d \t %s \t %s" % (int(time.time()), packageName, s)
#update apkInfo entry if infoDict updated (a new entry is also counted as updated) or current version apkDownloaded first time
if preInfoEntry != infoDict or (preIsCurrentVersionDownloaded == False
and isCurrentVersionDownloaded == True):
#apkInfo is collection for doc.details.appDetails, and also add isFree and isDownloaded
infoDict['isFree'] = isFree
#infoDict['isDownloaded'] only indicates whether we ever downloaded this apk.
#isCurrentVersionDownloaded indicates whether current version download succeeds
#It is possible previous round of cralwing did download this version, although current round fails to download the same version.
infoDict[
'isDownloaded'] = preIsDownloaded or isCurrentVersionDownloaded
infoDict['isCurrentVersionDownloaded'] = isCurrentVersionDownloaded
if preFileDir == "" and isCurrentVersionDownloaded:
#Only update fileDir when preFileDir is empty which means never downloaded
#first round crawling has one bug that all apps have a not empty fileDir
infoDict['fileDir'] = fileDir
else:
#Still using previous fileDir
infoDict['fileDir'] = preFileDir
#This is for static analysis purpose. Add the flag when current version apk is sucessfully downloaded and apk version updated or pre version has not been analyzed.
#Everytime only analyze db.apkInfo.find({'isApkUpdated': True})
#after analyze change the isApkUpdated to False
infoDict['isApkUpdated'] = preIsApkUpdated or (
isApkUpdated and isCurrentVersionDownloaded) or (
(not isApkUpdated) and (preIsCurrentVersionDownloaded == False)
and isCurrentVersionDownloaded == True)
if isSizeExceed != None:
infoDict['isSizeExceed'] = isSizeExceed
infoDict['updatedTimestamp'] = datetime.datetime.utcnow()
#even the download is not successful, if the appDetails is updated, the db entry will be updated
db.apkInfo.update({'packageName': packageName}, infoDict, upsert=True)
return ret
sys.exit(0)
packagename = sys.argv[1]
filename = None
if (len(sys.argv) == 3):
filename = sys.argv[2]
# read config from config.py
config = GooglePlayAPI.read_config()
# connect to GooglePlayStore
api = GooglePlayAPI(config['ANDROID_ID'])
api.login(config['GOOGLE_LOGIN'], config['GOOGLE_PASSWORD'], config['AUTH_TOKEN'])
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
if filename == None:
filename = "%s_%s.apk" % (packagename, vc)
# Download
print("Downloading to file %s with size %s..." % (filename, helpers.sizeof_fmt(doc.details.appDetails.installationSize)))
data = api.download(packagename, vc, ot)
io.open(filename, "wb").write(data)
print("Done")
# subcategory(title)
result['sub_title'] = ctr
# APK name
result['name'] = unicode(i.title).encode('utf8')
# print unicode(i.title).encode('utf8')
# rank
result['rank'] = rank
# print rank
# Packagename
result['pgname'] = i.details.appDetails.packageName
# print i.details.appDetails.packageName
# APK version
result['version'] = i.details.appDetails.versionCode
# print i.details.appDetails.versionCode
# APK size
result['size'] = sizeof_fmt(i.details.appDetails.installationSize)
# print sizeof_fmt(i.details.appDetails.installationSize)
# Upload date
result['upload_date'] = datetime.strptime(unicode(i.details.appDetails.uploadDate).encode('utf8'),
locale_timestring[LANG]).strftime('%Y-%m-%d')
# print datetime.strptime(unicode(i.details.appDetails.uploadDate).encode('utf8'),
# "%Y\345\271\264%m\346\234\210%d\346\227\245").strftime('%Y-%m-%d')
# Download APK
result['apkdata'] = api.download(i.details.appDetails.packageName,
i.details.appDetails.versionCode,
i.offer[0].offerType)
result.update(File(result['apkdata']).result)
rank += 1
try:
print "Download an app."
print "If filename is not present, will write to packagename.apk."
sys.exit(0)
packagename = sys.argv[1]
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
#print m
# Download
print "Downloading pkg : %s size: %s version: %s" % (
packagename, sizeof_fmt(doc.details.appDetails.installationSize),
doc.details.appDetails.versionString)
#data = api.download(packagename, vc, ot)
#open(filename, "wb").write(data)
api.download2(packagename, vc, filename, ot)
print "Done"
if (len(sys.argv) < 2):
print "Usage: %s packagename [filename]"
print "Download an app."
print "If filename is not present, will write to packagename.apk."
sys.exit(0)
packagename = sys.argv[1]
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
# Connect
api = GooglePlayAPI("3eded73a231f2845")
api.login("*****@*****.**", "gruegrue", AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def download_apk_via_url(apk):
GOOGLE_LOGIN = "******"
GOOGLE_PASSWORD = "******"
AUTH_TOKEN = "aAX_7XGsaT4B5hIDvtdh8oM-R7B5iidTCgHxXOKQXXFDFdmdzPcXKslTr6SX0YpiDMIORA."
f = 0
head, sep, tail = apk.partition('&')
apk = head
msg = ""
if "." in apk:
if (apk.startswith('http')):
if "play.google.com/store/apps/details?id=" in apk:
print "Link:" + apk
link, packagename = apk.split("=")
else:
message = "Wrong URL!"
return message
else:
packagename = apk
else:
message = "Invalid Package Name"
return message
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
print("Package Name: " + packagename)
# Connect
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
size = sizeof_fmt(doc.details.appDetails.installationSize)
l = len(size)
ext = size[(l - 2):l]
size = size[:-2]
if ext == "MB":
f = float(size)
if ext == "GB":
f = float(size) * 1024
if f > float(200):
message = "App Size is Greater than 200MB. Please Enter a Different URL or Package Name"
return message
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data, msg = api.download(packagename, vc, ot)
DIR = settings.BASE_DIR
ANAL_DIR = os.path.join(DIR)
fullpath = ANAL_DIR + '/uploads/' + packagename + ".apk"
open(fullpath, "wb").write(data)
print "APK Downloaded!"
return fullpath
except Exception as e:
print "Not able to fetch the apk: {}".format(e)
if "Google Play purchases are not supported in your country" in msg:
message = """App region isn't supported by the platform."""
elif "does not support this purchase." in msg:
message = """The platform doesn't support paid app download."""
else:
message = "Sorry! We are not able to fetch the APK at this moment. Please upload the APK."
return message
def plot_packets(batch_id):
"""
Plots time on the x axis vs length on the y axis
"""
dump_batch_names = get_fnames_by_id(batch_id)
if len(dump_batch_names) < len(PROC_ARGS):
print "At least one protocol was missing from batch %s" % batch_id
return
dump_protocols = []
filesize = 0
# Convert all the times to an offset from 0
xy = []
for name in sorted(dump_batch_names):
with open("packet_dumps/%s" % name) as f:
dump = json.load(f)
# If any of the dumps don't have a packets array, abort the whole dump
if not dump.get("stored_packets", True):
print "Dump %s was not captured with --store-packets" % name
return
dump_protocols.append(dump["protocol"])
filesize = sizeof_fmt(dump["file_size"])
x = [
datetime.strptime(l["time"], TIME_FORMAT) for l in dump["packets"]
]
t0 = min(x)
dt = timedelta(hours=t0.hour,
minutes=t0.minute,
seconds=t0.second,
microseconds=t0.microsecond)
x = [t - dt for t in x]
y = [l["length"] for l in dump["packets"]]
xy.append((x, y))
fig, axes = plt.subplots(len(PROC_ARGS), sharex="col")
max_yticks = 4
for i in range(len(PROC_ARGS)):
axes[i].plot(xy[i][0], xy[i][1], ".")
axes[i].xaxis.set_major_formatter(
dates.DateFormatter(PLOT_TIME_FORMAT))
axes[i].set_title(dump_protocols[i], size=16)
axes[i].yaxis.set_major_locator(plt.MaxNLocator(max_yticks))
# Add a line to show where the transfer stopped
ylim = axes[i].get_ylim()
end = max(xy[i][0])
axes[i].plot([end, end], [ylim[0], ylim[1]],
"g--",
linewidth=2,
label="final packet")
fig.text(0.5, 0.04, "Time elapsed", ha='center', va='center', size=16)
fig.text(0.04,
0.5,
"Length of payload (bytes)",
ha='center',
va='center',
rotation='vertical',
size=16)
fig.suptitle("Individual Packets for %s Transfer" % filesize, size=18)
axes[len(PROC_ARGS) - 1].legend(bbox_to_anchor=(1, -0.1))
fig.show()
print "If filename is not present, will write to packagename.apk."
sys.exit(0)
packagename = sys.argv[1]
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
logging.info( "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),)
data = api.download(packagename, vc, ot)
logging.info( "..."+filename)
#filepath=os.path.abspath(os.path.join(os.getcwd(), os.pardir)) +"\Download\\"
filepath = os.getcwd() +"/Download/"
logging.info( "Saving to"+filepath+filename)
open(filepath+filename, "wb").write(data)
logging.info( "Done with downloading apk")
exit(1)
doc = m.docV2
vc = doc.details.appDetails.versionCode
# Set the version code
if (len(sys.argv) == 3):
vc = int(sys.argv[2])
else:
vc = doc.details.appDetails.versionCode
# Genarate the filename with version code
if (len(sys.argv) == 4):
filename = sys.argv[3] + ".vc" + str(vc) + ".apk"
else:
filename = packagename + ".vc" + str(vc) + ".apk"
ot = doc.offer[0].offerType
# Download
print "versionCode:%d Downloading %s..." % (vc,
sizeof_fmt(doc.details.appDetails.installationSize),)
data = api.download(packagename, vc, ot)
if data == False:
print 'error in api.download'
sys.exit(1)
open(filename, "wb").write(data)
print "Done"
def downloadApkAndUpdateDB(api, db, packagename, fileDir):
filename = fileDir + '/' + packagename + ".apk"
try:
m = api.details(packagename)
except:
sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename))
traceback.print_exc(file=sys.stderr)
return (False, traceback.format_exc())
doc = m.docV2
vc = doc.details.appDetails.versionCode
# Zero length offer means the app is not found, possibly removed from the
# play store.
# We treat this as a success case so we can treat this app as processed
# and avoid re-downloading it if we need to recover from an exception.
if (len(doc.offer) == 0):
sys.stdout.write("%d \t %s \t Not Found\n" % (int(time.time()), packagename))
return (True, packagename + " Not Found")
try:
ot = doc.offer[0].offerType
except:
sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename))
print >> sys.stderr, doc
traceback.print_exc(file=sys.stderr)
return (False, traceback.format_exc())
packageName = doc.details.appDetails.packageName
#use eval, since only with api.toDict, pymongo will throw some warning related to wsgi
docDict = eval(str(api.toDict(doc)))
isApkUpdated = False
#apkDetails is collection for doc + updated timestamp
#if docDict does change, update the db entry
#use more query instead of insertion to speed up
#Do not get fields not in docDict from apkDetails
preDetailsEntry = db.apkDetails.find_one({'details.appDetails.packageName': packageName}, {'updatedTimestamp':0, '_id':0})
if preDetailsEntry != docDict:
#Warning: sometimes versionCode is not available
#versionCode is used for determine whether apk has been updated
#http://developer.android.com/tools/publishing/versioning.html
try:
isApkUpdated = (not preDetailsEntry) or (docDict['details']['appDetails']['versionCode'] != preDetailsEntry['details']['appDetails']['versionCode'])
except KeyError as e:
isApkUpdated = True
docDict['updatedTimestamp'] = datetime.datetime.utcnow()
db.apkDetails.update({'details.appDetails.packageName': packageName}, docDict, upsert=True)
else:
isApkUpdated = False
infoDict = docDict['details']['appDetails']
isFree = not doc.offer[0].checkoutFlowRequired
isCurrentVersionDownloaded = False
# 2015-04-17: Removed apk size restriction.
# According to
# http://android-developers.blogspot.com/2012/03/android-apps-break-50mb-barrier.html
# Each apk file will be at most 50 mb. An app that is listed as bigger than
# 50 mb on the playstore will have expansion packs that can supply
# additional things like graphics, etc.
# http://developer.android.com/distribute/googleplay/publish/preparing.html#size
isSizeExceed = False
# if doc.details.appDetails.installationSize > 52428800:
# isSizeExceed = True
# else:
# isSizeExceed = False
#Remove db entry fields which are not in infoDict
preInfoEntry = db.apkInfo.find_one({'packageName': packageName}, {'isFree':0, 'isSizeExceed': 0, 'updatedTimestamp':0, '_id':0})
if preInfoEntry == None:
preInfoEntry = {}
preIsDownloaded = preInfoEntry.pop('isDownloaded', False)
preIsCurrentVersionDownloaded = preInfoEntry.pop('isCurrentVersionDownloaded', False)
preIsApkUpdated = preInfoEntry.pop('isApkUpdated', False)
preFileDir = preInfoEntry.pop('fileDir', '')
# Default return value
ret = (True, packageName)
# Download when it is free and not exceed 50 mb and (current version in apkInfo was not downloaded or app has been updated since last time update apkInfo version)
if isFree and (not isSizeExceed) and ((not preIsCurrentVersionDownloaded) or isApkUpdated):
try:
data = api.download(packageName, vc, ot)
except Exception as e:
sys.stderr.write("%d \t %s\n" % (int(time.time()), packageName))
traceback.print_exc(file=sys.stderr)
isCurrentVersionDownloaded = False
ret = (False, traceback.format_exc())
else:
print "%d \t %s" % (int(time.time()), packageName)
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
if preFileDir != '':
fileDir = preFileDir
filename = preFileDir + '/' + packagename + ".apk"
open(filename, "wb").write(data)
print "Done"
isCurrentVersionDownloaded = True
else:
if isApkUpdated == False and preIsCurrentVersionDownloaded == True:
isCurrentVersionDownloaded = True
s = "Escape downloading isFree: %s, isSizeExceed: %s, preIsCurrentVersionDownloaded: %s, isApkUpdated: %s"%( isFree, isSizeExceed, preIsCurrentVersionDownloaded, isApkUpdated)
print "%d \t %s \t %s" % (int(time.time()), packageName, s)
#update apkInfo entry if infoDict updated (a new entry is also counted as updated) or current version apkDownloaded first time
if preInfoEntry != infoDict or (preIsCurrentVersionDownloaded == False and isCurrentVersionDownloaded == True):
#apkInfo is collection for doc.details.appDetails, and also add isFree and isDownloaded
infoDict['isFree'] = isFree
#infoDict['isDownloaded'] only indicates whether we ever downloaded this apk.
#isCurrentVersionDownloaded indicates whether current version download succeeds
#It is possible previous round of cralwing did download this version, although current round fails to download the same version.
infoDict['isDownloaded'] = preIsDownloaded or isCurrentVersionDownloaded
infoDict['isCurrentVersionDownloaded'] = isCurrentVersionDownloaded
if preFileDir == "" and isCurrentVersionDownloaded:
#Only update fileDir when preFileDir is empty which means never downloaded
#first round crawling has one bug that all apps have a not empty fileDir
infoDict['fileDir'] = fileDir
else:
#Still using previous fileDir
infoDict['fileDir'] = preFileDir
#This is for static analysis purpose. Add the flag when current version apk is sucessfully downloaded and apk version updated or pre version has not been analyzed.
#Everytime only analyze db.apkInfo.find({'isApkUpdated': True})
#after analyze change the isApkUpdated to False
infoDict['isApkUpdated'] = preIsApkUpdated or (isApkUpdated and isCurrentVersionDownloaded) or ((not isApkUpdated) and (preIsCurrentVersionDownloaded == False) and isCurrentVersionDownloaded == True)
if isSizeExceed != None:
infoDict['isSizeExceed'] = isSizeExceed
infoDict['updatedTimestamp'] = datetime.datetime.utcnow()
#even the download is not successful, if the appDetails is updated, the db entry will be updated
db.apkInfo.update({'packageName': packageName}, infoDict, upsert=True)
return ret
