def run(packagename): if (len(sys.argv) == 3): filename = sys.argv[2] else: filename = packagename + ".apk" # Connect api = GooglePlayAPI(config.ANDROID_ID) api.login(config.GOOGLE_LOGIN, config.GOOGLE_PASSWORD, config.AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt( doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def download(self, packageName): """Download an app and return its raw data (APK file). packageName is the app unique ID (usually starting with 'com.'). versionCode can be grabbed by using the details() method on the given app.""" # Get the version code and the offer type from the app details doc = self.details(packageName).docV2 versionCode = doc.details.appDetails.versionCode offerType = doc.offer[0].offerType path = "purchase" data = "ot=%d&doc=%s&vc=%d" % (offerType, packageName, versionCode) message = self.executeRequestApi2(path, data) url = message.payload.buyResponse.purchaseStatusResponse.appDeliveryData.downloadUrl cookie = message.payload.buyResponse.purchaseStatusResponse.appDeliveryData.downloadAuthCookie[0] cookies = { str(cookie.name): str(cookie.value) # python-requests #459 fixes this } headers = { "User-Agent" : "AndroidDownloadManager/4.1.1 (Linux; U; Android 4.1.1; Nexus S Build/JRO03E)", "Accept-Encoding": "", } print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), response = requests.get(url, headers=headers, cookies=cookies, verify=False) return response.content
def getDetailsForDownload(api, app): m = api.details(app) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType sz = sizeof_fmt(doc.details.appDetails.installationSize) return vc, ot, sz
def plot_speed_efficiency(df, filesize, title=True, legend=True, yaxis=True, xaxis=True): """ Given a pandas Dataframe, plots each protocol's speed vs it's data efficiency """ rows = df.loc[int(filesize)] x = rows["Bytes Total"] y = rows["Speed (bytes/s)"] for i in range(len(rows)): pname = rows.index[i] plt.scatter(x[i], y[i], marker=MARKERS[i], color=COLORS[i], s=150, label=pname) # Add a line to represent filesize ylim = plt.ylim() plt.plot([filesize, filesize], [ylim[0], ylim[1]], "r--", linewidth=3, label="filesize") # Add surrounding information if yaxis: plt.ylabel("Speed (bytes/s)", size=24) if xaxis: plt.xlabel("Total Bytes Transferred", size=24) if title: plt.title("Speed vs Data Efficiency for %s File" % sizeof_fmt(filesize), size=26, y=1.1) if legend: plt.legend(bbox_to_anchor=(1.3, 1), scatterpoints=1, prop={"size": "20"}) # Make sure the limits are correct and consistent plt.axes().yaxis.set_major_locator(plt.MaxNLocator(4)) plt.axes().xaxis.set_major_locator(plt.MaxNLocator(4)) plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0)) plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) plt.ylim(ylim) plt.show()
def plot_packets(batch_id): """ Plots time on the x axis vs length on the y axis """ dump_batch_names = get_fnames_by_id(batch_id) if len(dump_batch_names) < len(PROC_ARGS): print "At least one protocol was missing from batch %s" % batch_id return dump_protocols = [] filesize = 0 # Convert all the times to an offset from 0 xy = [] for name in sorted(dump_batch_names): with open("packet_dumps/%s" % name) as f: dump = json.load(f) # If any of the dumps don't have a packets array, abort the whole dump if not dump.get("stored_packets", True): print "Dump %s was not captured with --store-packets" % name return dump_protocols.append(dump["protocol"]) filesize = sizeof_fmt(dump["file_size"]) x = [datetime.strptime(l["time"], TIME_FORMAT) for l in dump["packets"]] t0 = min(x) dt = timedelta(hours=t0.hour, minutes=t0.minute, seconds=t0.second, microseconds=t0.microsecond) x = [t-dt for t in x] y = [l["length"] for l in dump["packets"]] xy.append((x, y)) fig, axes = plt.subplots(len(PROC_ARGS), sharex="col") max_yticks = 4 for i in range(len(PROC_ARGS)): axes[i].plot(xy[i][0], xy[i][1], ".") axes[i].xaxis.set_major_formatter(dates.DateFormatter(PLOT_TIME_FORMAT)) axes[i].set_title(dump_protocols[i], size=16) axes[i].yaxis.set_major_locator(plt.MaxNLocator(max_yticks)) # Add a line to show where the transfer stopped ylim = axes[i].get_ylim() end = max(xy[i][0]) axes[i].plot([end, end], [ylim[0], ylim[1]], "g--", linewidth=2, label="final packet") fig.text(0.5, 0.04, "Time elapsed", ha='center', va='center', size=16) fig.text(0.04, 0.5, "Length of payload (bytes)", ha='center', va='center', rotation='vertical', size=16) fig.suptitle("Individual Packets for %s Transfer" % filesize, size=18) axes[len(PROC_ARGS)-1].legend(bbox_to_anchor=(1, -0.1)) fig.show()
def scrape(id, package): packagename = package res = urllib2.urlopen("https://play.google.com/store/apps/details?id=%s&hl=en" % packagename) html = res.read() path = "assets/%d/" % id if not os.path.exists(path): os.makedirs(path) name = html.split('itemprop="name"')[1].split("</div>")[0].split("<div>")[1] desc = html.split('<div class="show-more-content text-body" itemprop="description">')[1].split( '<div class="show-more-end">' )[0] rating = html.split("Rated ")[1].split(" stars")[0] category = html.split('<span itemprop="genre">')[1].split("</span>")[0] stuff = [] for line in html.split("</div>"): if "full-screenshot" in line: url = line.split('src="')[1][:-3] stuff.append(url) x = 0 for img in stuff[1:]: print img urllib.urlretrieve(img, "%s%d.webp" % (path, x)) x += 1 filename = path + packagename + ".apk" # Connect api = GooglePlayAPI(ANDROID_ID) api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done" ap = apk.APK(filename) badging = os.popen("/home/thomas/dev/android-odroid/out/host/linux-x86/bin/aapt dump badging %s" % filename).read() for line in badging.split("\n"): if "launchable-activity" in line: activity = line.split("'")[1] return [name, desc, rating, package, activity, category]
def downloadApps(api,packagename,filename): # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def downloadApps(api, packagename, filename): # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt( doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def download_apk(packagename, filename): # Connect api = GooglePlayAPI(ANDROID_ID) api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def downloadApps(api,packagename,filename): # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType if Desc==True: getDescriptions(doc) print if Perm==True: getPermissions(doc) #print doc.details.appDetails.permission # Download if Down==True: print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def get_apk(packagename, friendly_name): # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType easy_name = friendly_name + "-" + str(vc) filename = easy_name +".apk" # logic to check if we have this version here if do_we_already_have(friendly_name, easy_name): print "We already have " + easy_name + " skipping download" else: print "Downloading " + easy_name + " from the Google Play Store." # Download print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def plot_speed_efficiency(df, filesize, title=True, legend=True, yaxis=True, xaxis=True): """ Given a pandas Dataframe, plots each protocol's speed vs it's data efficiency """ rows = df.loc[int(filesize)] x = rows["Bytes Total"] y = rows["Speed (bytes/s)"] for i in range(len(rows)): pname = rows.index[i] plt.scatter(x[i], y[i], marker=MARKERS[i], color=COLORS[i], s=150, label=pname) # Add a line to represent filesize ylim = plt.ylim() plt.plot([filesize, filesize], [ylim[0], ylim[1]], "r--", linewidth=3, label="filesize") # Add surrounding information if yaxis: plt.ylabel("Speed (bytes/s)", size=24) if xaxis: plt.xlabel("Total Bytes Transferred", size=24) if title: plt.title("Speed vs Data Efficiency for %s File" % sizeof_fmt(filesize), size=26, y=1.1) if legend: plt.legend(bbox_to_anchor=(1.3, 1), scatterpoints=1, prop={"size":"20"}) # Make sure the limits are correct and consistent plt.axes().yaxis.set_major_locator(plt.MaxNLocator(4)) plt.axes().xaxis.set_major_locator(plt.MaxNLocator(4)) plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0)) plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) plt.ylim(ylim) plt.show()
if ((len(sys.argv) >= 3) and (len(sys.argv)<5)): filename = sys.argv[2] google_pass = sys.argv[3] elif (len(sys.argv)<3): filename = packagename + ".apk" google_pass = getpass.getpass() else: print "Usage: %s packagename filename, password]" print "Download an app." print "If filename is not present, will write to packagename.apk." print "If password is not present, user will be prompted for one" sys.exit(0) # Connect api = GooglePlayAPI(ANDROID_ID) api.login(GOOGLE_LOGIN, google_pass, AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def downloadApkAndUpdateDB(api, db, packagename, fileDir): filename = fileDir + '/' + packagename + ".apk" try: m = api.details(packagename) except: sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename)) traceback.print_exc(file=sys.stderr) return (False, traceback.format_exc()) doc = m.docV2 vc = doc.details.appDetails.versionCode # Zero length offer means the app is not found, possibly removed from the # play store. # We treat this as a success case so we can treat this app as processed # and avoid re-downloading it if we need to recover from an exception. if (len(doc.offer) == 0): sys.stdout.write("%d \t %s \t Not Found\n" % (int(time.time()), packagename)) return (True, packagename + " Not Found") try: ot = doc.offer[0].offerType except: sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename)) print >> sys.stderr, doc traceback.print_exc(file=sys.stderr) return (False, traceback.format_exc()) packageName = doc.details.appDetails.packageName #use eval, since only with api.toDict, pymongo will throw some warning related to wsgi docDict = eval(str(api.toDict(doc))) isApkUpdated = False #apkDetails is collection for doc + updated timestamp #if docDict does change, update the db entry #use more query instead of insertion to speed up #Do not get fields not in docDict from apkDetails preDetailsEntry = db.apkDetails.find_one( {'details.appDetails.packageName': packageName}, { 'updatedTimestamp': 0, '_id': 0 }) if preDetailsEntry != docDict: #Warning: sometimes versionCode is not available #versionCode is used for determine whether apk has been updated #http://developer.android.com/tools/publishing/versioning.html try: isApkUpdated = (not preDetailsEntry) or ( docDict['details']['appDetails']['versionCode'] != preDetailsEntry['details']['appDetails']['versionCode']) except KeyError as e: isApkUpdated = True docDict['updatedTimestamp'] = datetime.datetime.utcnow() db.apkDetails.update({'details.appDetails.packageName': packageName}, docDict, upsert=True) else: isApkUpdated = False infoDict = docDict['details']['appDetails'] isFree = not doc.offer[0].checkoutFlowRequired isCurrentVersionDownloaded = False # 2015-04-17: Removed apk size restriction. # According to # http://android-developers.blogspot.com/2012/03/android-apps-break-50mb-barrier.html # Each apk file will be at most 50 mb. An app that is listed as bigger than # 50 mb on the playstore will have expansion packs that can supply # additional things like graphics, etc. # http://developer.android.com/distribute/googleplay/publish/preparing.html#size isSizeExceed = False # if doc.details.appDetails.installationSize > 52428800: # isSizeExceed = True # else: # isSizeExceed = False #Remove db entry fields which are not in infoDict preInfoEntry = db.apkInfo.find_one({'packageName': packageName}, { 'isFree': 0, 'isSizeExceed': 0, 'updatedTimestamp': 0, '_id': 0 }) if preInfoEntry == None: preInfoEntry = {} preIsDownloaded = preInfoEntry.pop('isDownloaded', False) preIsCurrentVersionDownloaded = preInfoEntry.pop( 'isCurrentVersionDownloaded', False) preIsApkUpdated = preInfoEntry.pop('isApkUpdated', False) preFileDir = preInfoEntry.pop('fileDir', '') # Default return value ret = (True, packageName) # Download when it is free and not exceed 50 mb and (current version in apkInfo was not downloaded or app has been updated since last time update apkInfo version) if isFree and (not isSizeExceed) and ((not preIsCurrentVersionDownloaded) or isApkUpdated): try: data = api.download(packageName, vc, ot) except Exception as e: sys.stderr.write("%d \t %s\n" % (int(time.time()), packageName)) traceback.print_exc(file=sys.stderr) isCurrentVersionDownloaded = False ret = (False, traceback.format_exc()) else: print "%d \t %s" % (int(time.time()), packageName) print "Downloading %s..." % sizeof_fmt( doc.details.appDetails.installationSize), if preFileDir != '': fileDir = preFileDir filename = preFileDir + '/' + packagename + ".apk" open(filename, "wb").write(data) print "Done" isCurrentVersionDownloaded = True else: if isApkUpdated == False and preIsCurrentVersionDownloaded == True: isCurrentVersionDownloaded = True s = "Escape downloading isFree: %s, isSizeExceed: %s, preIsCurrentVersionDownloaded: %s, isApkUpdated: %s" % ( isFree, isSizeExceed, preIsCurrentVersionDownloaded, isApkUpdated) print "%d \t %s \t %s" % (int(time.time()), packageName, s) #update apkInfo entry if infoDict updated (a new entry is also counted as updated) or current version apkDownloaded first time if preInfoEntry != infoDict or (preIsCurrentVersionDownloaded == False and isCurrentVersionDownloaded == True): #apkInfo is collection for doc.details.appDetails, and also add isFree and isDownloaded infoDict['isFree'] = isFree #infoDict['isDownloaded'] only indicates whether we ever downloaded this apk. #isCurrentVersionDownloaded indicates whether current version download succeeds #It is possible previous round of cralwing did download this version, although current round fails to download the same version. infoDict[ 'isDownloaded'] = preIsDownloaded or isCurrentVersionDownloaded infoDict['isCurrentVersionDownloaded'] = isCurrentVersionDownloaded if preFileDir == "" and isCurrentVersionDownloaded: #Only update fileDir when preFileDir is empty which means never downloaded #first round crawling has one bug that all apps have a not empty fileDir infoDict['fileDir'] = fileDir else: #Still using previous fileDir infoDict['fileDir'] = preFileDir #This is for static analysis purpose. Add the flag when current version apk is sucessfully downloaded and apk version updated or pre version has not been analyzed. #Everytime only analyze db.apkInfo.find({'isApkUpdated': True}) #after analyze change the isApkUpdated to False infoDict['isApkUpdated'] = preIsApkUpdated or ( isApkUpdated and isCurrentVersionDownloaded) or ( (not isApkUpdated) and (preIsCurrentVersionDownloaded == False) and isCurrentVersionDownloaded == True) if isSizeExceed != None: infoDict['isSizeExceed'] = isSizeExceed infoDict['updatedTimestamp'] = datetime.datetime.utcnow() #even the download is not successful, if the appDetails is updated, the db entry will be updated db.apkInfo.update({'packageName': packageName}, infoDict, upsert=True) return ret
sys.exit(0) packagename = sys.argv[1] filename = None if (len(sys.argv) == 3): filename = sys.argv[2] # read config from config.py config = GooglePlayAPI.read_config() # connect to GooglePlayStore api = GooglePlayAPI(config['ANDROID_ID']) api.login(config['GOOGLE_LOGIN'], config['GOOGLE_PASSWORD'], config['AUTH_TOKEN']) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType if filename == None: filename = "%s_%s.apk" % (packagename, vc) # Download print("Downloading to file %s with size %s..." % (filename, helpers.sizeof_fmt(doc.details.appDetails.installationSize))) data = api.download(packagename, vc, ot) io.open(filename, "wb").write(data) print("Done")
# subcategory(title) result['sub_title'] = ctr # APK name result['name'] = unicode(i.title).encode('utf8') # print unicode(i.title).encode('utf8') # rank result['rank'] = rank # print rank # Packagename result['pgname'] = i.details.appDetails.packageName # print i.details.appDetails.packageName # APK version result['version'] = i.details.appDetails.versionCode # print i.details.appDetails.versionCode # APK size result['size'] = sizeof_fmt(i.details.appDetails.installationSize) # print sizeof_fmt(i.details.appDetails.installationSize) # Upload date result['upload_date'] = datetime.strptime(unicode(i.details.appDetails.uploadDate).encode('utf8'), locale_timestring[LANG]).strftime('%Y-%m-%d') # print datetime.strptime(unicode(i.details.appDetails.uploadDate).encode('utf8'), # "%Y\345\271\264%m\346\234\210%d\346\227\245").strftime('%Y-%m-%d') # Download APK result['apkdata'] = api.download(i.details.appDetails.packageName, i.details.appDetails.versionCode, i.offer[0].offerType) result.update(File(result['apkdata']).result) rank += 1 try:
print "Download an app." print "If filename is not present, will write to packagename.apk." sys.exit(0) packagename = sys.argv[1] if (len(sys.argv) == 3): filename = sys.argv[2] else: filename = packagename + ".apk" # Connect api = GooglePlayAPI(ANDROID_ID) api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType #print m # Download print "Downloading pkg : %s size: %s version: %s" % ( packagename, sizeof_fmt(doc.details.appDetails.installationSize), doc.details.appDetails.versionString) #data = api.download(packagename, vc, ot) #open(filename, "wb").write(data) api.download2(packagename, vc, filename, ot) print "Done"
if (len(sys.argv) < 2): print "Usage: %s packagename [filename]" print "Download an app." print "If filename is not present, will write to packagename.apk." sys.exit(0) packagename = sys.argv[1] if (len(sys.argv) == 3): filename = sys.argv[2] else: filename = packagename + ".apk" # Connect api = GooglePlayAPI("3eded73a231f2845") api.login("*****@*****.**", "gruegrue", AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download print "Downloading %s..." % sizeof_fmt( doc.details.appDetails.installationSize), data = api.download(packagename, vc, ot) open(filename, "wb").write(data) print "Done"
def download_apk_via_url(apk): GOOGLE_LOGIN = "******" GOOGLE_PASSWORD = "******" AUTH_TOKEN = "aAX_7XGsaT4B5hIDvtdh8oM-R7B5iidTCgHxXOKQXXFDFdmdzPcXKslTr6SX0YpiDMIORA." f = 0 head, sep, tail = apk.partition('&') apk = head msg = "" if "." in apk: if (apk.startswith('http')): if "play.google.com/store/apps/details?id=" in apk: print "Link:" + apk link, packagename = apk.split("=") else: message = "Wrong URL!" return message else: packagename = apk else: message = "Invalid Package Name" return message if (len(sys.argv) == 3): filename = sys.argv[2] else: filename = packagename + ".apk" print("Package Name: " + packagename) # Connect requests.packages.urllib3.disable_warnings(InsecureRequestWarning) api = GooglePlayAPI(ANDROID_ID) api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN) try: # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType size = sizeof_fmt(doc.details.appDetails.installationSize) l = len(size) ext = size[(l - 2):l] size = size[:-2] if ext == "MB": f = float(size) if ext == "GB": f = float(size) * 1024 if f > float(200): message = "App Size is Greater than 200MB. Please Enter a Different URL or Package Name" return message # Download print "Downloading %s..." % sizeof_fmt( doc.details.appDetails.installationSize), data, msg = api.download(packagename, vc, ot) DIR = settings.BASE_DIR ANAL_DIR = os.path.join(DIR) fullpath = ANAL_DIR + '/uploads/' + packagename + ".apk" open(fullpath, "wb").write(data) print "APK Downloaded!" return fullpath except Exception as e: print "Not able to fetch the apk: {}".format(e) if "Google Play purchases are not supported in your country" in msg: message = """App region isn't supported by the platform.""" elif "does not support this purchase." in msg: message = """The platform doesn't support paid app download.""" else: message = "Sorry! We are not able to fetch the APK at this moment. Please upload the APK." return message
def plot_packets(batch_id): """ Plots time on the x axis vs length on the y axis """ dump_batch_names = get_fnames_by_id(batch_id) if len(dump_batch_names) < len(PROC_ARGS): print "At least one protocol was missing from batch %s" % batch_id return dump_protocols = [] filesize = 0 # Convert all the times to an offset from 0 xy = [] for name in sorted(dump_batch_names): with open("packet_dumps/%s" % name) as f: dump = json.load(f) # If any of the dumps don't have a packets array, abort the whole dump if not dump.get("stored_packets", True): print "Dump %s was not captured with --store-packets" % name return dump_protocols.append(dump["protocol"]) filesize = sizeof_fmt(dump["file_size"]) x = [ datetime.strptime(l["time"], TIME_FORMAT) for l in dump["packets"] ] t0 = min(x) dt = timedelta(hours=t0.hour, minutes=t0.minute, seconds=t0.second, microseconds=t0.microsecond) x = [t - dt for t in x] y = [l["length"] for l in dump["packets"]] xy.append((x, y)) fig, axes = plt.subplots(len(PROC_ARGS), sharex="col") max_yticks = 4 for i in range(len(PROC_ARGS)): axes[i].plot(xy[i][0], xy[i][1], ".") axes[i].xaxis.set_major_formatter( dates.DateFormatter(PLOT_TIME_FORMAT)) axes[i].set_title(dump_protocols[i], size=16) axes[i].yaxis.set_major_locator(plt.MaxNLocator(max_yticks)) # Add a line to show where the transfer stopped ylim = axes[i].get_ylim() end = max(xy[i][0]) axes[i].plot([end, end], [ylim[0], ylim[1]], "g--", linewidth=2, label="final packet") fig.text(0.5, 0.04, "Time elapsed", ha='center', va='center', size=16) fig.text(0.04, 0.5, "Length of payload (bytes)", ha='center', va='center', rotation='vertical', size=16) fig.suptitle("Individual Packets for %s Transfer" % filesize, size=18) axes[len(PROC_ARGS) - 1].legend(bbox_to_anchor=(1, -0.1)) fig.show()
print "If filename is not present, will write to packagename.apk." sys.exit(0) packagename = sys.argv[1] if (len(sys.argv) == 3): filename = sys.argv[2] else: filename = packagename + ".apk" # Connect api = GooglePlayAPI(ANDROID_ID) api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN) # Get the version code and the offer type from the app details m = api.details(packagename) doc = m.docV2 vc = doc.details.appDetails.versionCode ot = doc.offer[0].offerType # Download logging.info( "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),) data = api.download(packagename, vc, ot) logging.info( "..."+filename) #filepath=os.path.abspath(os.path.join(os.getcwd(), os.pardir)) +"\Download\\" filepath = os.getcwd() +"/Download/" logging.info( "Saving to"+filepath+filename) open(filepath+filename, "wb").write(data) logging.info( "Done with downloading apk")
exit(1) doc = m.docV2 vc = doc.details.appDetails.versionCode # Set the version code if (len(sys.argv) == 3): vc = int(sys.argv[2]) else: vc = doc.details.appDetails.versionCode # Genarate the filename with version code if (len(sys.argv) == 4): filename = sys.argv[3] + ".vc" + str(vc) + ".apk" else: filename = packagename + ".vc" + str(vc) + ".apk" ot = doc.offer[0].offerType # Download print "versionCode:%d Downloading %s..." % (vc, sizeof_fmt(doc.details.appDetails.installationSize),) data = api.download(packagename, vc, ot) if data == False: print 'error in api.download' sys.exit(1) open(filename, "wb").write(data) print "Done"
def downloadApkAndUpdateDB(api, db, packagename, fileDir): filename = fileDir + '/' + packagename + ".apk" try: m = api.details(packagename) except: sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename)) traceback.print_exc(file=sys.stderr) return (False, traceback.format_exc()) doc = m.docV2 vc = doc.details.appDetails.versionCode # Zero length offer means the app is not found, possibly removed from the # play store. # We treat this as a success case so we can treat this app as processed # and avoid re-downloading it if we need to recover from an exception. if (len(doc.offer) == 0): sys.stdout.write("%d \t %s \t Not Found\n" % (int(time.time()), packagename)) return (True, packagename + " Not Found") try: ot = doc.offer[0].offerType except: sys.stderr.write("%d \t %s\n" % (int(time.time()), packagename)) print >> sys.stderr, doc traceback.print_exc(file=sys.stderr) return (False, traceback.format_exc()) packageName = doc.details.appDetails.packageName #use eval, since only with api.toDict, pymongo will throw some warning related to wsgi docDict = eval(str(api.toDict(doc))) isApkUpdated = False #apkDetails is collection for doc + updated timestamp #if docDict does change, update the db entry #use more query instead of insertion to speed up #Do not get fields not in docDict from apkDetails preDetailsEntry = db.apkDetails.find_one({'details.appDetails.packageName': packageName}, {'updatedTimestamp':0, '_id':0}) if preDetailsEntry != docDict: #Warning: sometimes versionCode is not available #versionCode is used for determine whether apk has been updated #http://developer.android.com/tools/publishing/versioning.html try: isApkUpdated = (not preDetailsEntry) or (docDict['details']['appDetails']['versionCode'] != preDetailsEntry['details']['appDetails']['versionCode']) except KeyError as e: isApkUpdated = True docDict['updatedTimestamp'] = datetime.datetime.utcnow() db.apkDetails.update({'details.appDetails.packageName': packageName}, docDict, upsert=True) else: isApkUpdated = False infoDict = docDict['details']['appDetails'] isFree = not doc.offer[0].checkoutFlowRequired isCurrentVersionDownloaded = False # 2015-04-17: Removed apk size restriction. # According to # http://android-developers.blogspot.com/2012/03/android-apps-break-50mb-barrier.html # Each apk file will be at most 50 mb. An app that is listed as bigger than # 50 mb on the playstore will have expansion packs that can supply # additional things like graphics, etc. # http://developer.android.com/distribute/googleplay/publish/preparing.html#size isSizeExceed = False # if doc.details.appDetails.installationSize > 52428800: # isSizeExceed = True # else: # isSizeExceed = False #Remove db entry fields which are not in infoDict preInfoEntry = db.apkInfo.find_one({'packageName': packageName}, {'isFree':0, 'isSizeExceed': 0, 'updatedTimestamp':0, '_id':0}) if preInfoEntry == None: preInfoEntry = {} preIsDownloaded = preInfoEntry.pop('isDownloaded', False) preIsCurrentVersionDownloaded = preInfoEntry.pop('isCurrentVersionDownloaded', False) preIsApkUpdated = preInfoEntry.pop('isApkUpdated', False) preFileDir = preInfoEntry.pop('fileDir', '') # Default return value ret = (True, packageName) # Download when it is free and not exceed 50 mb and (current version in apkInfo was not downloaded or app has been updated since last time update apkInfo version) if isFree and (not isSizeExceed) and ((not preIsCurrentVersionDownloaded) or isApkUpdated): try: data = api.download(packageName, vc, ot) except Exception as e: sys.stderr.write("%d \t %s\n" % (int(time.time()), packageName)) traceback.print_exc(file=sys.stderr) isCurrentVersionDownloaded = False ret = (False, traceback.format_exc()) else: print "%d \t %s" % (int(time.time()), packageName) print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize), if preFileDir != '': fileDir = preFileDir filename = preFileDir + '/' + packagename + ".apk" open(filename, "wb").write(data) print "Done" isCurrentVersionDownloaded = True else: if isApkUpdated == False and preIsCurrentVersionDownloaded == True: isCurrentVersionDownloaded = True s = "Escape downloading isFree: %s, isSizeExceed: %s, preIsCurrentVersionDownloaded: %s, isApkUpdated: %s"%( isFree, isSizeExceed, preIsCurrentVersionDownloaded, isApkUpdated) print "%d \t %s \t %s" % (int(time.time()), packageName, s) #update apkInfo entry if infoDict updated (a new entry is also counted as updated) or current version apkDownloaded first time if preInfoEntry != infoDict or (preIsCurrentVersionDownloaded == False and isCurrentVersionDownloaded == True): #apkInfo is collection for doc.details.appDetails, and also add isFree and isDownloaded infoDict['isFree'] = isFree #infoDict['isDownloaded'] only indicates whether we ever downloaded this apk. #isCurrentVersionDownloaded indicates whether current version download succeeds #It is possible previous round of cralwing did download this version, although current round fails to download the same version. infoDict['isDownloaded'] = preIsDownloaded or isCurrentVersionDownloaded infoDict['isCurrentVersionDownloaded'] = isCurrentVersionDownloaded if preFileDir == "" and isCurrentVersionDownloaded: #Only update fileDir when preFileDir is empty which means never downloaded #first round crawling has one bug that all apps have a not empty fileDir infoDict['fileDir'] = fileDir else: #Still using previous fileDir infoDict['fileDir'] = preFileDir #This is for static analysis purpose. Add the flag when current version apk is sucessfully downloaded and apk version updated or pre version has not been analyzed. #Everytime only analyze db.apkInfo.find({'isApkUpdated': True}) #after analyze change the isApkUpdated to False infoDict['isApkUpdated'] = preIsApkUpdated or (isApkUpdated and isCurrentVersionDownloaded) or ((not isApkUpdated) and (preIsCurrentVersionDownloaded == False) and isCurrentVersionDownloaded == True) if isSizeExceed != None: infoDict['isSizeExceed'] = isSizeExceed infoDict['updatedTimestamp'] = datetime.datetime.utcnow() #even the download is not successful, if the appDetails is updated, the db entry will be updated db.apkInfo.update({'packageName': packageName}, infoDict, upsert=True) return ret