def main(input_file, out_dir):
total = file_len(input_file)
fail_counter = FailCounter(50)
proxy_iterator = ProxyIterator(config.proxy_list)
proxy_iterator.next()
file_object = open(input_file)
try:
input_text = file_object.read()
finally:
file_object.close()
try:
data = json.loads(input_text)
assert data[0]['hl']
except Exception:
print '读取json输入文件[ %s ]出错...\n' % (input_file)
exit(0)
lang = data[0]['hl']
api = GooglePlayAPI(ANDROID_ID, lang)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
now = datetime.datetime.now()
out_file = '%s%s_reviewDesc-%s.txt' % (out_dir,
lang,
now.strftime('%Y%m%d-%H:%M:%S'))
progress = {'total' : len(data), 'current' : 0}
for i in xrange(len(data)):
progress['current'] = progress['current'] + 1
request = data[i]['app_name'].strip()
print '[ %s ] starting...' % (request)
if not request:
continue
for i in xrange(len(config.proxy_list)):
stat = list_top_downloads(request, api, out_file, progress)
if not stat:
print '切换代理重试中...'
fail_counter.append(False)
proxy_iterator.next()
if fail_counter.fail_rate() > 0.7:
time.sleep(60)
fail_counter.reset()
proxy_iterator.move_to_first()
api = GooglePlayAPI(ANDROID_ID, lang=lang)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
else:
fail_counter.append(True)
break
def getPermissions(packageName, multiple=False):
api = GooglePlayAPI(config.ANDROID_ID)
api.login(config.GOOGLE_LOGIN, config.GOOGLE_PASSWORD, config.AUTH_TOKEN)
result = {}
# Only one app
if (not multiple):
response = api.details(packageName)
permissions = []
values = response.docV2.details.appDetails.permission._values
permissions = [x.encode('utf-8') for x in values]
name = response.docV2.title
result["name"] = name.encode("utf8")
result["permissions"] = permissions
logo = response.docV2.image._values[0]
image = logo.imageUrl.encode("utf8")
result["logo"] = image
return result
else: # More than one app, not tested or used
response = api.bulkDetails(packageName)
for entry in response.entry:
if (not not entry.ListFields()): # if the entry is not empty
result[
entry.doc.docid] = entry.doc.details.appDetails.permission
return result
def run(packagename):
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
# Connect
api = GooglePlayAPI(config.ANDROID_ID)
api.login(config.GOOGLE_LOGIN, config.GOOGLE_PASSWORD, config.AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def getPackagePermission(packagenames,androidId):
api = GooglePlayAPI(androidId)
'''
Prajit Das: Cycling through the different androidIds we have
to ensure that we are not blocked by Google
The following statement was the original code
'''
#api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Only one app
if (len(packagenames) == 1):
response = api.details(packagenames[0])
permissionList = []
#print "\n".join(i.encode('utf8') for i in response.docV2.details.appDetails.permission)
for permission in response.docV2.details.appDetails.permission:
permissionList.append(permission.encode('utf8'))
for permission in permissionList: print permission
return permissionList
else: # More than one app
permissionListDict = {}
response = api.bulkDetails(packagenames)
for entry in response.entry:
if (not not entry.ListFields()): # if the entry is not empty
permissionList = []
# print entry.doc.docid + ":"
# print "\n".join(" "+i.encode('utf8') for i in entry.doc.details.appDetails.permission)
for permission in entry.doc.details.appDetails.permission:
permissionList.append(permission.encode('utf8'))
for permission in permissionList: print permission
permissionListDict[entry.doc.docid] = permissionList
return permissionListDict
def login():
api = None
try:
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
except Exception as e:
print e
return api
def connect(android_id, google_login, google_password, auth_token):
api = GooglePlayAPI(android_id)
try:
api.login(google_login, google_password, auth_token)
except:
print >> sys.stderr, int(time.time())
traceback.print_exc(file=sys.stderr)
return api
def download_sdk_to_file(packagename, filename):
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Download
data = api.download(packagename)
with open(filename, "wb") as f:
f.write(data)
def login(self, retry=0):
try_count = 0
while True:
if retry > 0 and try_count >= retry:
return
try:
api = GooglePlayAPI(self.android_id, lang=self.lang)
api.login(self.google_login, self.google_password,
self.auth_token)
return api
except Exception as e:
print "GooglePlayAPI Login Faild, Has Try Count:%s" % try_count, e
try_count += 1
def main(lang, input_file, out_file, task_tag=''):
total = file_len(input_file)
ez_dump = EzDumper(out_file, total, '%s : download_num' % task_tag)
fail_counter = FailCounter(50)
proxy_iterator = ProxyIterator(config.proxy_list)
proxy_iterator.next()
api = GooglePlayAPI(ANDROID_ID, lang=lang)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
with open(input_file, 'r') as f:
current = 0
for line in f:
current += 1
request = line.strip()
if not request:
continue
ez_dump.dump(current, "KEYWORD:[%s]" % request)
for i in xrange(len(config.proxy_list)):
stat = list_top_downloads(request, ez_dump, api)
if stat:
fail_counter.append(True)
break
else:
fail_counter.append(False)
ez_dump.dump(None,
"fail_rate:[%s]" % fail_counter.fail_rate())
proxy_iterator.next()
if fail_counter.fail_rate() > 0.7:
ez_dump.dump(None, 'sleep for %s seconds ...' % 60)
time.sleep(60)
fail_counter.reset()
proxy_iterator.move_to_first()
api = GooglePlayAPI(ANDROID_ID, lang=lang)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
def get_categories():
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
response = api.browse()
categories = []
# print SEPARATOR.join(["ID", "Name"])
for c in response.category:
categories.append(
SEPARATOR.join(
i.encode('utf8')
for i in [urlparse.parse_qs(c.dataUrl)['cat'][0]]))
# print SEPARATOR.join(i.encode('utf8') for i in [urlparse.parse_qs(c.dataUrl)['cat'][0], c.name])
return categories
def download_apk(packagename, filename):
# Connect
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def findAppInfo(packageName):
request = packageName
nb_res = None
offset = None
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.search(request, nb_res, offset)
doc = message.doc[0]
for c in doc.child:
if c.docid.startswith(packageName):
result = {}
result["creator"] = c.creator
result["price"] = c.offer[0].formattedAmount
return result
except Exception, e:
print str(e)
def list(cat, ctr=None, nb_results=None, offset=None):
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.list(cat, ctr, nb_results, offset)
except:
print "Error: HTTP 500 - one of the provided parameters is invalid"
if (ctr is None):
print SEPARATOR.join(["Subcategory ID", "Name"])
for doc in message.doc:
print SEPARATOR.join(
[doc.docid.encode('utf8'),
doc.title.encode('utf8')])
else:
print_header_line()
doc = message.doc[0]
results = []
for c in doc.child:
results.append(get_parsed_result(c))
return results
def crawlAll(id, offset=0):
# declared as global in order for changes to be acknowledged by every threads
global FOUND_APP
global LOCK
global TOTAL_SIZE
global THREAD_PROGRESS
api = GooglePlayAPI(ANDROID_ID[id])
api.login(GOOGLE_LOGIN[id], GOOGLE_PASSWORD[id], AUTH_TOKEN)
user = id
for wordId in range(offset, len(SPLITED_DICO[id])):
try:
message = api.search(SPLITED_DICO[id][wordId],
250) # Search the playstore with the word
except:
print "Error: HTTP 500 - one of the provided parameters is invalid"
return 0
try:
doc = message.doc[0]
except IndexError: # if we were blocked
print "Blocked, switching Account"
connected = 0
while (connected == 0):
user += THREAD_NUMBER
if user >= len(ANDROID_ID):
user = id
print GOOGLE_LOGIN[user]
time.sleep(4) # Wait a bit before connecting again
del api
api = GooglePlayAPI(ANDROID_ID[user], LANG)
try:
api.login(GOOGLE_LOGIN[user], GOOGLE_PASSWORD[user],
None) # attempt another connection
connected = 1
except LoginError:
connected = 0
print "Auth error, retrying ..."
continue
for c in doc.child:
try:
#print_result_line(c)
package = c.docid
while LOCK == 1: # prevent reading while other thread is writing
pass
if package in FOUND_APP: # if app was found in a previous search
continue
name = c.title
length = c.details.appDetails.installationSize / float(
1024 * 1024)
TOTAL_SIZE += length
l = [package, name, str(length) + "MB"]
print SEPARATOR.join(unicode(i).encode('utf8') for i in l)
while LOCK == 1: # prevent writing when other thread is writing
pass
LOCK = 1 # Acquire lock
FOUND_APP.append(package) # commit change
LOCK = 0 # release lock
print "\nNumber Of app = " + str(len(FOUND_APP))
print "\nAverage size Of app = " + str(
TOTAL_SIZE / len(FOUND_APP))
if length > 0: # if apk is more than 1 MB
print 'too large, skip'
continue
folder = DOWNLOAD_PATH
if not os.path.exists(folder):
os.mkdir(folder)
filename = folder + '/' + name + '.apk'
if not os.path.isfile(filename):
downloadApps(api, package, filename)
except:
continue
THREAD_PROGRESS[id] = wordId
print "processed words by thread #" + str(id) + ": " + str(
THREAD_PROGRESS[id])
return FOUND_APP
config = json.loads(
config_file.read().strip()
)
try:
deviceid = config["deviceid"]
email = config["email"]
password = config["password"]
except KeyError:
logger.critical('Config problem: %s', 'Missing config parameters')
sys.exit(1)
except IOError:
logger.critical('Config problem: %s', 'Missing config file')
sys.exit(1)
api = GooglePlayAPI(deviceid)
api.login(email, password)
logger.info('Conecting to host {}:{}'.format('localhost', '27017'))
client = pymongo.MongoClient("localhost", 27017)
db = client.test
SUB_CAT = (
'apps_topselling_paid',
'apps_topselling_free',
'apps_topgrossing',
'apps_topselling_new_paid',
'apps_topselling_new_free'
)
count = 0
print "subcategory: You can get a list of all subcategories available, by supplying a valid category"
sys.exit(0)
cat = sys.argv[1]
ctr = None
nb_results = None
offset = None
if (len(sys.argv) >= 3):
ctr = sys.argv[2]
if (len(sys.argv) >= 4):
nb_results = sys.argv[3]
if (len(sys.argv) == 5):
offset = sys.argv[4]
api = GooglePlayAPI()
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.list(cat, ctr, nb_results, offset)
except:
print "Error: HTTP 500 - one of the provided parameters is invalid"
if (ctr is None):
print SEPARATOR.join(["Subcategory ID", "Name"])
for doc in message.doc:
print SEPARATOR.join(
[doc.docid.encode('utf8'),
doc.title.encode('utf8')])
else:
print_header_line()
if (len(sys.argv) < 2):
print "Usage: %s packagename [filename]"
print "Download an app."
print "If filename is not present, will write to packagename.apk."
sys.exit(0)
packagename = sys.argv[1]
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
# Connect
api = GooglePlayAPI("3eded73a231f2845")
api.login("*****@*****.**", "gruegrue", AUTH_TOKEN)
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data = api.download(packagename, vc, ot)
open(filename, "wb").write(data)
print "Done"
def __init__(self):
super(GoogleAnalysis, self).__init__()
self.api = GooglePlayAPI(ANDROID_ID)
self.api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
def retrieveCategoryPackageNames(request):
nb_res = None
offset = None
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
packageNamesList = []
#print_header_line()
# continue requesting until receive ALL apps by category. use to make histogram.
while True:
try:
message = api.search(request, nb_res, offset)
except:
break
if message.doc:
doc = message.doc[0]
for c in doc.child:
packageNamesList.append(c.docid)
offset = len(packageNamesList)
else:
break
dic = {}
final_results = {}
# Only one app returned TODO update this
if (len(packageNamesList) == 1):
response = api.details(packageNamesList[0])
print "\n".join(i.encode('utf8') for i in response.docV2.details.appDetails.permission)
# More than one app
else:
response = api.bulkDetails(packageNamesList)
for entry in response.entry:
hasCustom = False
if (entry.ListFields()):
iconUrl = ""
for x in entry.doc.image:
if x.imageType == 4:
iconUrl = x.imageUrl
final_results[entry.doc.docid] = {'package': entry.doc.docid, 'title':entry.doc.title,
'creator':entry.doc.creator,
'price':{'amount':entry.doc.offer[0].formattedAmount, 'currency':entry.doc.offer[0].currencyCode},
'icon':iconUrl,
'permissions':[perm for perm in entry.doc.details.appDetails.permission],
'rating':{'stars':entry.doc.aggregateRating.starRating, 'count':entry.doc.aggregateRating.ratingsCount},
'shareUrl': entry.doc.shareUrl
}
for permission in entry.doc.details.appDetails.permission:
# apps can have custom permissions that don't start with android.permission.PERM_NAME_HERE
# are they safe? Will warn users about custom permissions.
perm = permission.split(".")
if (permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and (len(perm) !=3 or perm[0] != "android" or perm[1] != "permission")):
hasCustom = True
continue # don't want to count custom permissions in the histogram dictionary since they will all of them will be unusual
#perm = perm[-2] + "." + perm[-1]
if permission in dic:
dic[permission] += 1;
else:
dic[permission] = 1;
final_results[entry.doc.docid]['custom'] = hasCustom
#print dic
DANGEROUS_PERMS = ["android.permission.ACCESS_COARSE_LOCATION",
"android.permission.ACCESS_FINE_LOCATION",
"android.permission.ACCESS_MOCK_LOCATION",
"android.permission.AUTHENTICATE_ACCOUNTS",
"android.permission.BLUETOOTH",
"android.permission.BLUETOOTH_ADMIN",
"android.permission.CALL_PHONE",
"android.permission.CAMERA",
"android.permission.CHANGE_CONFIGURATION",
"android.permission.CHANGE_NETWORK_STATE",
"android.permission.CHANGE_WIFI_MULTICAST_STATE",
"android.permission.CHANGE_WIFI_STATE",
"android.permission.CLEAR_APP_CACHE",
"android.permission.DUMP",
"android.permission.GET_TASKS",
"android.permission.INTERNET",
"android.permission.MANAGE_ACCOUNTS",
"android.permission.MODIFY_AUDIO_SETTINGS",
"android.permission.MODIFY_PHONE_STATE",
"android.permission.MOUNT_FORMAT_FILESYSTEMS",
"android.permission.MOUNT_UNMOUNT_FILESYSTEMS",
"android.permission.PERSISTENT_ACTIVITY",
"android.permission.PROCESS_OUTGOING_CALLS",
"android.permission.READ_CALENDAR",
"android.permission.READ_CONTACTS",
"android.permission.READ_LOGS",
"android.permission.READ_OWNER_DATA",
"android.permission.READ_PHONE_STATE",
"android.permission.READ_SMS",
"android.permission.READ_USER_DICTIONARY",
"android.permission.RECEIVE_MMS",
"android.permission.RECEIVE_SMS",
"android.permission.RECEIVE_WAP_PUSH",
"android.permission.RECORD_AUDIO",
"android.permission.REORDER_TASKS",
"android.permission.SEND_SMS",
"android.permission.SET_ALWAYS_FINISH",
"android.permission.SET_ANIMATION_SCALE",
"android.permission.SET_DEBUG_APP",
"android.permission.SET_PROCESS_LIMIT",
"android.permission.SET_TIME_ZONE",
"android.permission.SIGNAL_PERSISTENT_PROCESSES",
"android.permission.SUBSCRIBED_FEEDS_WRITE",
"android.permission.SYSTEM_ALERT_WINDOW",
"android.permission.USE_CREDENTIALS",
"android.permission.WAKE_LOCK",
"android.permission.WRITE_APN_SETTINGS",
"android.permission.WRITE_CALENDAR",
"android.permission.WRITE_CONTACTS",
"android.permission.WRITE_EXTERNAL_STORAGE",
"android.permission.WRITE_OWNER_DATA",
"android.permission.WRITE_SETTINGS",
"android.permission.WRITE_SMS",
"android.permission.WRITE_SYNC_SETTINGS",
"com.android.browser.permission.READ_HISTORY_BOOKMARKS",
"com.android.browser.permission.WRITE_HISTORY_BOOKMARKS"]
#go through final_results and add field for unusual permissions
for docid in final_results:
final_results[docid]['unusual'] = False
final_results[docid]['dangerous'] = False
for permission in final_results[docid]['permissions']:
perm = permission.split(".")
if (permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and (len(perm) !=3 or perm[0] != "android" or perm[1] != "permission")):
continue
elif dic[permission] < 0.05*offset:
final_results[docid]['unusual'] = True
if 'unusual_perms' in final_results[docid]:
final_results[docid]['unusual_perms'].append(permission)
else:
final_results[docid]['unusual_perms'] = [permission]
if permission in DANGEROUS_PERMS:
final_results[docid]['dangerous'] = True
if 'dangerous_permissions' in final_results[docid]:
final_results[docid]['dangerous_permissions'].append(permission)
else:
final_results[docid]['dangerous_permissions'] = [permission]
final = json.dumps({"app": final_results.values()})
print final
return None
#!/usr/bin/python
# Do not remove
GOOGLE_LOGIN = GOOGLE_PASSWORD = AUTH_TOKEN = None
import requests
import sys
import urlparse
from pprint import pprint
from google.protobuf import text_format
from config import *
from googleplay import GooglePlayAPI
# Ignore unverified HTTPS request warning.
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
response = api.browse()
print SEPARATOR.join(["ID", "Name"])
for c in response.category:
print SEPARATOR.join(
i.encode('utf8')
for i in [urlparse.parse_qs(c.dataUrl)['cat'][0], c.name])
def download_apk_via_url(apk):
GOOGLE_LOGIN = "******"
GOOGLE_PASSWORD = "******"
AUTH_TOKEN = "aAX_7XGsaT4B5hIDvtdh8oM-R7B5iidTCgHxXOKQXXFDFdmdzPcXKslTr6SX0YpiDMIORA."
f = 0
head, sep, tail = apk.partition('&')
apk = head
msg = ""
if "." in apk:
if (apk.startswith('http')):
if "play.google.com/store/apps/details?id=" in apk:
print "Link:" + apk
link, packagename = apk.split("=")
else:
message = "Wrong URL!"
return message
else:
packagename = apk
else:
message = "Invalid Package Name"
return message
if (len(sys.argv) == 3):
filename = sys.argv[2]
else:
filename = packagename + ".apk"
print("Package Name: " + packagename)
# Connect
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
# Get the version code and the offer type from the app details
m = api.details(packagename)
doc = m.docV2
vc = doc.details.appDetails.versionCode
ot = doc.offer[0].offerType
size = sizeof_fmt(doc.details.appDetails.installationSize)
l = len(size)
ext = size[(l - 2):l]
size = size[:-2]
if ext == "MB":
f = float(size)
if ext == "GB":
f = float(size) * 1024
if f > float(200):
message = "App Size is Greater than 200MB. Please Enter a Different URL or Package Name"
return message
# Download
print "Downloading %s..." % sizeof_fmt(
doc.details.appDetails.installationSize),
data, msg = api.download(packagename, vc, ot)
DIR = settings.BASE_DIR
ANAL_DIR = os.path.join(DIR)
fullpath = ANAL_DIR + '/uploads/' + packagename + ".apk"
open(fullpath, "wb").write(data)
print "APK Downloaded!"
return fullpath
except Exception as e:
print "Not able to fetch the apk: {}".format(e)
if "Google Play purchases are not supported in your country" in msg:
message = """App region isn't supported by the platform."""
elif "does not support this purchase." in msg:
message = """The platform doesn't support paid app download."""
else:
message = "Sorry! We are not able to fetch the APK at this moment. Please upload the APK."
return message
