def main():
search_term = sys.argv[1]
'''
apps will be a list of dicts describing each returned app:
{'app_name':<app name>,'app_id':<app id>,'app_creator':<creator>,'app_permissions':[list]}
'''
apps = []
nb_res = 100 # apps to retrieve. API allows for max of 100.
offset = 0
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.search(search_term, nb_res, offset)
except:
print "Error: something went wrong. Google may be throttling or rejecting the request."
sys.exit(1)
doc = message.doc[0]
for c in doc.child:
permissions = []
details = api.details(c.docid)
for line in details.docV2.details.appDetails.permission:
permissions.append(line)
apps.append({'app_name':c.title,'app_id':c.docid,'app_creator':c.creator,'app_permissions':permissions})
'''
We are interested in the set of all possible permissions that start with 'ANDROID.'
'''
permissions = Set([])
for app in apps:
for permission in app["app_permissions"]:
if permission.upper()[0:8] == "ANDROID.":
permissions.add(permission.upper())
'''
Create ARFF output for Weka
'''
dataset = open(search_term + ".arff",'w')
dataset.write("@relation Appdata\n")
dataset.write("@attribute index NUMERIC\n")
dataset.write("@attribute app_name STRING\n")
for att in permissions:
dataset.write("@attribute "+ att + " {0,1}\n")
dataset.write("@data\n")
i = 0 # index for cross-referencing
for app in apps:
print("{} {}").format(str(i), str(app)) # index
perm_str = str(i) + ',' + app['app_id'] + ','
app_perm_upper = []
for app_permission in app['app_permissions']:
app_perm_upper.append(app_permission.upper())
for permission in permissions:
if permission in app_perm_upper:
perm_str = perm_str + '1,'
else:
perm_str = perm_str + '0,'
dataset.write(perm_str[:-1])
dataset.write("\n")
i += 1
def findAppInfo(packageName):
request = packageName
nb_res = None
offset = None
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.search(request, nb_res, offset)
doc = message.doc[0]
for c in doc.child:
if c.docid.startswith(packageName):
result = {}
result["creator"] = c.creator
result["price"] = c.offer[0].formattedAmount
return result
except Exception, e:
print str(e)
def findAppInfo(packageName):
request = packageName
nb_res = None
offset = None
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.search(request, nb_res, offset)
doc = message.doc[0]
for c in doc.child:
if c.docid.startswith(packageName):
result = {}
result["creator"] = c.creator
result["price"] = c.offer[0].formattedAmount
return result
except Exception, e:
print str(e)
print "Usage: %s request [nb_results] [offset]" % sys.argv[0]
print "Search for an app."
print "If request contains a space, don't forget to surround it with \"\""
sys.exit(0)
request = sys.argv[1]
nb_res = None
offset = None
if (len(sys.argv) >= 3):
nb_res = int(sys.argv[2])
if (len(sys.argv) >= 4):
offset = int(sys.argv[3])
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.search(request, nb_res, offset)
except:
print "Error: something went wrong. Maybe the nb_res you specified was too big?"
sys.exit(1)
print_header_line()
print message
doc = message.doc[0]
for c in doc.child:
print_result_line(c)
NB_RES = int(sys.argv[2])
if (len(sys.argv) >= 4):
OFFSET = int(sys.argv[3])
# Check request content
print "Request:", request
print "Number of request:", NB_RES
print "Offset:", OFFSET
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
print "NB_RES/OFFSET:",NB_RES/OFFSET
for i in range (50):
message = api.search(request, str(NB_RES), str(i*NB_RES))
doc = message.doc[0]
for c in doc.child:
print c
l = [
# unicode type
c.docid,
c.title,
c.creator,
c.descriptionHtml, # need to remove control characters
c.offer[0].formattedAmount,
c.details.appDetails.versionCode, # long type
c.details.appDetails.versionString, #unicode type
c.details.appDetails.appCategory, # class type
c.details.appDetails.installationSize, # long type
def crawlAll(id, offset=0):
# declared as global in order for changes to be acknowledged by every threads
global FOUND_APP
global LOCK
global TOTAL_SIZE
global THREAD_PROGRESS
api = GooglePlayAPI(ANDROID_ID[id])
api.login(GOOGLE_LOGIN[id], GOOGLE_PASSWORD[id], AUTH_TOKEN)
user = id
for wordId in range(offset, len(SPLITED_DICO[id])):
try:
message = api.search(SPLITED_DICO[id][wordId],
250) # Search the playstore with the word
except:
print "Error: HTTP 500 - one of the provided parameters is invalid"
return 0
try:
doc = message.doc[0]
except IndexError: # if we were blocked
print "Blocked, switching Account"
connected = 0
while (connected == 0):
user += THREAD_NUMBER
if user >= len(ANDROID_ID):
user = id
print GOOGLE_LOGIN[user]
time.sleep(4) # Wait a bit before connecting again
del api
api = GooglePlayAPI(ANDROID_ID[user], LANG)
try:
api.login(GOOGLE_LOGIN[user], GOOGLE_PASSWORD[user],
None) # attempt another connection
connected = 1
except LoginError:
connected = 0
print "Auth error, retrying ..."
continue
for c in doc.child:
try:
#print_result_line(c)
package = c.docid
while LOCK == 1: # prevent reading while other thread is writing
pass
if package in FOUND_APP: # if app was found in a previous search
continue
name = c.title
length = c.details.appDetails.installationSize / float(
1024 * 1024)
TOTAL_SIZE += length
l = [package, name, str(length) + "MB"]
print SEPARATOR.join(unicode(i).encode('utf8') for i in l)
while LOCK == 1: # prevent writing when other thread is writing
pass
LOCK = 1 # Acquire lock
FOUND_APP.append(package) # commit change
LOCK = 0 # release lock
print "\nNumber Of app = " + str(len(FOUND_APP))
print "\nAverage size Of app = " + str(
TOTAL_SIZE / len(FOUND_APP))
if length > 0: # if apk is more than 1 MB
print 'too large, skip'
continue
folder = DOWNLOAD_PATH
if not os.path.exists(folder):
os.mkdir(folder)
filename = folder + '/' + name + '.apk'
if not os.path.isfile(filename):
downloadApps(api, package, filename)
except:
continue
THREAD_PROGRESS[id] = wordId
print "processed words by thread #" + str(id) + ": " + str(
THREAD_PROGRESS[id])
return FOUND_APP
if (len(sys.argv) < 2):
print "Usage: %s request [nb_results] [offset]" % sys.argv[0]
print "Search for an app."
print "If request contains a space, don't forget to surround it with \"\""
sys.exit(0)
request = sys.argv[1]
nb_res = None
offset = None
if (len(sys.argv) >= 3):
nb_res = int(sys.argv[2])
if (len(sys.argv) >= 4):
offset = int(sys.argv[3])
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
try:
message = api.search(request, nb_res, offset)
except:
print "Error: something went wrong. Maybe the nb_res you specified was too big?"
sys.exit(1)
print_header_line()
doc = message.doc[0]
for c in doc.child:
print_result_line(c)
def retrieveCategoryPackageNames(request):
nb_res = None
offset = None
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
packageNamesList = []
#print_header_line()
# continue requesting until receive ALL apps by category. use to make histogram.
while True:
try:
message = api.search(request, nb_res, offset)
except:
break
if message.doc:
doc = message.doc[0]
for c in doc.child:
packageNamesList.append(c.docid)
offset = len(packageNamesList)
else:
break
dic = {}
final_results = {}
# Only one app returned TODO update this
if (len(packageNamesList) == 1):
response = api.details(packageNamesList[0])
print "\n".join(i.encode('utf8') for i in response.docV2.details.appDetails.permission)
# More than one app
else:
response = api.bulkDetails(packageNamesList)
for entry in response.entry:
hasCustom = False
if (entry.ListFields()):
iconUrl = ""
for x in entry.doc.image:
if x.imageType == 4:
iconUrl = x.imageUrl
final_results[entry.doc.docid] = {'package': entry.doc.docid, 'title':entry.doc.title,
'creator':entry.doc.creator,
'price':{'amount':entry.doc.offer[0].formattedAmount, 'currency':entry.doc.offer[0].currencyCode},
'icon':iconUrl,
'permissions':[perm for perm in entry.doc.details.appDetails.permission],
'rating':{'stars':entry.doc.aggregateRating.starRating, 'count':entry.doc.aggregateRating.ratingsCount},
'shareUrl': entry.doc.shareUrl
}
for permission in entry.doc.details.appDetails.permission:
# apps can have custom permissions that don't start with android.permission.PERM_NAME_HERE
# are they safe? Will warn users about custom permissions.
perm = permission.split(".")
if (permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and (len(perm) !=3 or perm[0] != "android" or perm[1] != "permission")):
hasCustom = True
continue # don't want to count custom permissions in the histogram dictionary since they will all of them will be unusual
#perm = perm[-2] + "." + perm[-1]
if permission in dic:
dic[permission] += 1;
else:
dic[permission] = 1;
final_results[entry.doc.docid]['custom'] = hasCustom
#print dic
DANGEROUS_PERMS = ["android.permission.ACCESS_COARSE_LOCATION",
"android.permission.ACCESS_FINE_LOCATION",
"android.permission.ACCESS_MOCK_LOCATION",
"android.permission.AUTHENTICATE_ACCOUNTS",
"android.permission.BLUETOOTH",
"android.permission.BLUETOOTH_ADMIN",
"android.permission.CALL_PHONE",
"android.permission.CAMERA",
"android.permission.CHANGE_CONFIGURATION",
"android.permission.CHANGE_NETWORK_STATE",
"android.permission.CHANGE_WIFI_MULTICAST_STATE",
"android.permission.CHANGE_WIFI_STATE",
"android.permission.CLEAR_APP_CACHE",
"android.permission.DUMP",
"android.permission.GET_TASKS",
"android.permission.INTERNET",
"android.permission.MANAGE_ACCOUNTS",
"android.permission.MODIFY_AUDIO_SETTINGS",
"android.permission.MODIFY_PHONE_STATE",
"android.permission.MOUNT_FORMAT_FILESYSTEMS",
"android.permission.MOUNT_UNMOUNT_FILESYSTEMS",
"android.permission.PERSISTENT_ACTIVITY",
"android.permission.PROCESS_OUTGOING_CALLS",
"android.permission.READ_CALENDAR",
"android.permission.READ_CONTACTS",
"android.permission.READ_LOGS",
"android.permission.READ_OWNER_DATA",
"android.permission.READ_PHONE_STATE",
"android.permission.READ_SMS",
"android.permission.READ_USER_DICTIONARY",
"android.permission.RECEIVE_MMS",
"android.permission.RECEIVE_SMS",
"android.permission.RECEIVE_WAP_PUSH",
"android.permission.RECORD_AUDIO",
"android.permission.REORDER_TASKS",
"android.permission.SEND_SMS",
"android.permission.SET_ALWAYS_FINISH",
"android.permission.SET_ANIMATION_SCALE",
"android.permission.SET_DEBUG_APP",
"android.permission.SET_PROCESS_LIMIT",
"android.permission.SET_TIME_ZONE",
"android.permission.SIGNAL_PERSISTENT_PROCESSES",
"android.permission.SUBSCRIBED_FEEDS_WRITE",
"android.permission.SYSTEM_ALERT_WINDOW",
"android.permission.USE_CREDENTIALS",
"android.permission.WAKE_LOCK",
"android.permission.WRITE_APN_SETTINGS",
"android.permission.WRITE_CALENDAR",
"android.permission.WRITE_CONTACTS",
"android.permission.WRITE_EXTERNAL_STORAGE",
"android.permission.WRITE_OWNER_DATA",
"android.permission.WRITE_SETTINGS",
"android.permission.WRITE_SMS",
"android.permission.WRITE_SYNC_SETTINGS",
"com.android.browser.permission.READ_HISTORY_BOOKMARKS",
"com.android.browser.permission.WRITE_HISTORY_BOOKMARKS"]
#go through final_results and add field for unusual permissions
for docid in final_results:
final_results[docid]['unusual'] = False
final_results[docid]['dangerous'] = False
for permission in final_results[docid]['permissions']:
perm = permission.split(".")
if (permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and permission != "com.android.browser.permission.WRITE_HISTORY_BOOKMARKS" and (len(perm) !=3 or perm[0] != "android" or perm[1] != "permission")):
continue
elif dic[permission] < 0.05*offset:
final_results[docid]['unusual'] = True
if 'unusual_perms' in final_results[docid]:
final_results[docid]['unusual_perms'].append(permission)
else:
final_results[docid]['unusual_perms'] = [permission]
if permission in DANGEROUS_PERMS:
final_results[docid]['dangerous'] = True
if 'dangerous_permissions' in final_results[docid]:
final_results[docid]['dangerous_permissions'].append(permission)
else:
final_results[docid]['dangerous_permissions'] = [permission]
final = json.dumps({"app": final_results.values()})
print final
return None
