class Thread_mongo(threading.Thread):
#thread class
def __init__(self,lock,api_key,data):
threading.Thread.__init__(self)
self.v = virustotal.VirusTotal(api_key)
self.db = DB()
self.lock = lock
self.thread_data = data
def run(self):
av_result = {}
for i in self.thread_data:
time.sleep(20)
result = self.v.get(i['md5'])
if result is not None:
av_result['summary'] = {}
av_result['summary']['total'] = result.total
av_result['summary']['positives'] = result.positives
for antivirus, malware in result:
av_result[antivirus[0]] = {}
av_result[antivirus[0]]['version'] = antivirus[1]
av_result[antivirus[0]]['result'] = malware
self.lock.acquire()
self.db.update_av_report(i['_id'], av_result)
self.lock.release()
else:
continue
def main():
try:
db = DB()
except:
logging.error("DB error")
raise
while(True):
doc = db.get_apk({'vt_scan': False, 'limit': 1})
if not doc:
logging.info("Maybe there's no document without vt_scan:true.")
break
av_result = vt().get(doc['md5'])
if av_result is None:
time.sleep(20)
filename = '/tmp/'+doc['pgname']+'.apk'
with open(filename, 'wb') as f:
f.write(db.get_apk_file(doc['apkdata']))
av_result = vt().submit_sample(filename)
os.remove(filename)
logging.info("Get av_result again")
# It will try to get report with several queries,
# so we take some sleep here.
time.sleep(60)
logging.debug("av_result: {}".format(av_result))
db.update_av_report(doc['_id'], av_result)
time.sleep(20)
def run(category_subpage):
#data = self.rawContent.findAll("div",{"class": "image-style-for-related-posts"})# 84 items
apk_info = {}
apk_info['appCategory'] = []
apk_info['source'] = source
apk_info['packageName'] = []
apk_info['apkName'] = []
fakeURL_list = []
pkg_name_list = []
dlink_list = []
result = {}
result['vt_scan'] = False
result['url_scan'] = False
result['source'] = 'androidapksfree.com/applications/apps'
#result['title']=category_subpage
for subpage_link in category_subpage:
subpage = BeautifulSoup(urllib.urlopen('https://' + subpage_link))
subpg = subpage.findAll("div",
{"class": "image-style-for-related-posts"})
for content in xrange(0, len(subpg)):
for url in subpg[content].findAll("a"):
try:
fakeURL = url.get('href')
fakeURL_list.append(fakeURL)
#print "fakeURL : ",fakeURL
apk_name = url.get('href')[36:][:-1]
print colored("downloading", 'red'), apk_name, "..."
pkg_name = _get_pkg_name(fakeURL)
pkg_name_list.append(pkg_name)
dlink = _get_d_link(fakeURL)
dlink_list.append(dlink)
apk_category = subpage_link[42:][:-1]
apk_info['appCategory'].append(apk_category)
apk_info['packageName'].append(pkg_name)
apk_info['apkName'].append(apk_name)
result['title'] = apk_category
result['pkgName'] = pkg_name
file_name = result['pkgName'] + '.apk'
result['name'] = file_name
result['timeStamp'] = timestamp
#print "apk_name : ",apk_name
#apkdata=_download_apk(dlink,apk_name)
#result['apkData'] = apkdata
DB().insert_apk(result)
DB().get_apk_info()
except Exception, e:
traceback.print_exc()
def download_apk():
"""Get document Objectid and Download APK file
"""
from core.db.Mongo import DB
from bson.objectid import ObjectId
my_db = DB()
apk_id = request.form['download_apk']
apk_info = my_db.get_apk({'_id': ObjectId(apk_id), 'limit': 1})
logging.debug(
'Download {}, {}'.format(apk_info['md5'], apk_info['apkdata']))
apkdata = my_db.get_apk_file(apk_info['apkdata'])
response = make_response(apkdata)
response.headers['Content-Type'] = 'application/vnd.android.package-archive'
response.headers['Content-Disposition'] = 'attachment; filename=' + apk_info['pgname'] + ".apk"
return response
def download_apk():
"""Get document Objectid and Download APK file
"""
from core.db.Mongo import DB
from bson.objectid import ObjectId
my_db = DB()
apk_id = request.form['download_apk']
apk_info = my_db.get_apk({'_id': ObjectId(apk_id), 'limit': 1})
logging.debug('Download {}, {}'.format(apk_info['md5'],
apk_info['apkdata']))
apkdata = my_db.get_apk_file(apk_info['apkdata'])
response = make_response(apkdata)
response.headers[
'Content-Type'] = 'application/vnd.android.package-archive'
response.headers[
'Content-Disposition'] = 'attachment; filename=' + apk_info[
'pgname'] + ".apk"
return response
def main():
start = time.time()
try:
db = DB()
except:
logging.error("DB error")
raise
number_of_key = 8
#get all data whitch including sacn :False
doc = db.get_all_vt_False()
lock = threading.Lock()#thread lock
thread_data = spilt(doc,doc.count(),number_of_key)
# api_key =['51d63dc8b2860fbd889ea73d564e361e1ec795ce2daadb1046771272336cdadf',
# '20f0728b711931ef2f60c8c403e83c20b600a902a12293a7d1fe566f85ca22dd',
# '7ec895bab30a273bf6df3e211105f5f2ee45a96ddea57f53d6e4fe2b98f0c7c1',
# 'd0fe387a075ca62d0336485641912f1b318240f6132c576fa96dbf81b242da71',
# '29b45a9dc40737a7bc894cbacc3da603044e7f3a2651606dfca89de9accab80a',
# '51d63dc8b2860fbd889ea73d564e361e1ec795ce2daadb1046771272336cdadf',
# '60473b7caf108d05a5f51b9fd7544f6bb7bd0a4d966ca58d0c7b65e43611abc9',
# '860011e025932bd8ad550e3174b75ee1c686134543a4635a4e37fef038c0fbec']
thread_pool = []
#deal apk with md5
for i in range(0,number_of_key):
p = Thread_mongo(lock,api_key[i],thread_data[i])
thread_pool.append(p)
for i in thread_pool:
i.start()
for i in thread_pool:
i.join()
# get the remaining data
doc = db.get_all_vt_False()
v = virustotal.VirusTotal('51d63dc8b2860fbd889ea73d564e361e1ec795ce2daadb1046771272336cdadf')
#send apk data by one process
for i in doc:
print i['name']
time.sleep(20)
filename = '/tmp/'+i['pgname']+'.apk'#write down the apk file in the disk
with open(filename, 'wb') as f:
f.write(db.get_apk_file(i['apkdata']))
av_result = submit_sample(v,filename)
os.remove(filename)
db.update_av_report(i['_id'], av_result)
end = time.time()
print 'total used:', end - start,' s'
def all_apks():
from core.db.Mongo import DB
# result = DB().get_apk({'title': 'APP_WIDGETS'})
result = []
g.total = list(DB().get_apk({}))
for i in g.total[(10 * (g.page - 1)):(10 * (g.page - 1)) + 10]:
result.append(i)
for i in g.total[0:20]:
logging.info(i['name'])
return render_template('all_apks.html',
apks=Pagination(g.page, len(g.total), result),
view='ui.all_apks',
**request.args.to_dict())
def Get_apk_name_and_link(apk_category):
result = {}
result['vt_scan'] = False
result['url_scan'] = False
result['source'] = 'apkpure.com'
result['title'] = apk_category
#Connect to the apk page
apk_category_link = "https://apkpure.com/"+ apk_category
res = Retry_session(apk_category_link,100)
#parser the page
soup = BeautifulSoup(res,"html.parser")
#find all apk's informations in this page
app_in_category = soup.find_all("div", attrs={"class": "category-template-title"})
#store the apk's name
app_name_set = []
app_urls_to_download = []
for app in app_in_category:
title_set = app.find_all(title=True)
app_name = title_set[0]['title'].replace(" ","_")
#replace the space to underline
app_name_set.append(app_name)
uri = app.find_all(href=True)
url_to_download = "https://apkpure.com" + uri[0]['href'] + '/download?from=category'
app_urls_to_download.append(url_to_download)
for i in range(len(app_name_set)):
Get_info_status = False
Get_download_status = False
try:
Get_info_status = Get_apk_information(app_in_category[i],result)
Get_download_status = Download_link(app_urls_to_download[i],app_name_set[i],i+1,result)
except:
traceback.print_exc()
pass
if Get_info_status is True and Get_download_status is True :
try:
DB().insert_apk(result)
except:
traceback.print_exc()
else:
pass
def main():
try:
db = DB()
except:
logging.error("DB error")
raise
while (True):
#find the first data with vt_scan is False
doc = db.get_apk({'vt_scan': False, 'limit': 1})
if not doc:
logging.info("Maybe there's no document without vt_scan:true.")
break
av_result = vt().get(doc['md5'])
if av_result is None:
time.sleep(20)
filename = '/tmp/' + doc[
'pgname'] + '.apk' #write down the apk file in the disk
with open(filename, 'wb') as f:
f.write(db.get_apk_file(doc['apkdata']))
av_result = vt().submit_sample(
filename) #send the file to vt for scan
os.remove(filename)
logging.info("Get av_result again")
# It will try to get report with several queries,
# so we take some sleep here.
time.sleep(60)
logging.debug("av_result: {}".format(av_result))
db.update_av_report(doc['_id'], av_result) # insert vt report into db
time.sleep(20)
def Get_apk_name_and_link(apk_category):
result = {}
result['vt_scan'] = False
result['url_scan'] = False
result['source'] = 'apkpure.com'
result['title'] = apk_category
#Connect to the apk page
apk_category_link = "https://apkpure.com/"+ apk_category
res = Retry_session(apk_category_link,100)
#parser the page
soup = BeautifulSoup(res,"html.parser")
#find all apk's informations in this page
app_in_category = soup.find_all("div", attrs={"class": "category-template-title"})
#store the apk's name
app_name_set = []
app_urls_to_download = []
for app in app_in_category:
title_set = app.find_all(title=True)
# app_name = title_set[0]['title'].replace(" ","_")
app_name = title_set[0]['title']
print "app_name : ",app_name
#replace the space to underline
app_name_set.append(app_name)
uri = app.find_all(href=True)
print "uri : ",uri
url_to_download = "https://apkpure.com" + uri[0]['href'] + '/download?from=category'
print "url_to_download : ",url_to_download
app_urls_to_download.append(url_to_download)
Bar_string = 'Processing '+ result['title']
bar = Bar(Bar_string, max = int(len(app_name_set)))
print "NameSet : "
for item in app_name_set:
print "item"
for i in range(len(app_name_set)):
Get_info_status = False
Get_download_status = False
try:
Get_info_status = Get_apk_information(app_in_category[i],result)
print "Get_info_status : ",Get_info_status
Get_download_status = Download_link(app_urls_to_download[i],app_name_set[i],i+1,result)
print "Get_download_status : ",Get_download_status
except:
traceback.print_exc()
pass
if Get_info_status is True and Get_download_status is True :
try:
DB().insert_apk(result)
except:
traceback.print_exc()
else:
pass
#write down which app was fail
# with open('Fail_app.txt', 'a') as f:
# f.write(result['title']+' '+result['name']+'\n')
# f.close
bar.next()
bar.finish()
def insert_into_db(result):
DB().insert_apk(result)
def daily_apks():
import datetime
from core.db.Mongo import DB
today = datetime.datetime.strftime(datetime.datetime.today(), '%Y-%m-%d')
# db_result_gplay = list(DB().get_apk({'submit_date': today, 'source': 'googleplay'}))
# db_result_mapk = list(DB().get_apk({'submit_date': today, 'source': 'mapk'}))
# db_result_gplay = list(DB().get_apk({'source': 'googleplay'}))
# db_result_mapk = list(DB().get_apk({"source": "mapk"}))
db_result_allapk = list(DB().get_apk({'submit_date': today}))
# db_result_mapk = list(DB().get_apk({'submit_date': today}))
# db_result_APKPure = list(DB().get_apk({'submit_date': today}))
# db_result_gplay = list(DB().get_apk({"$and":[{'submit_date': "2017-01-15"}, {'source': 'googleplay'}]}))
# db_result_mapk = list(DB().get_apk({"$and":[{'submit_date': "2017-01-15"}, {'source': 'mapk'}]}))
db_result_gplay_today = []
db_result_mapk_today = []
db_result_APKPure_today = []
for apk_record in db_result_allapk:
if apk_record['source'] == "googleplay":
db_result_gplay_today.append(apk_record)
elif apk_record['source'] == "mapk":
db_result_mapk_today.append(apk_record)
else:
db_result_APKPure_today.append(apk_record)
categories_result_gplay = daily_apks_by_categories(db_result_gplay_today)
antivirus_result_gplay = daily_apks_by_antivirus(db_result_gplay_today)
permission_result_gplay = daily_apks_by_permission(db_result_gplay_today)
URL_result_gplay = daily_apks_by_URL(db_result_gplay_today)
IP_result_gplay = daily_apks_by_IP(db_result_gplay_today)
categories_result_mapk = daily_apks_by_categories(db_result_mapk_today)
antivirus_result_mapk = daily_apks_by_antivirus(db_result_mapk_today)
permission_result_mapk = daily_apks_by_permission(db_result_mapk_today)
URL_result_mapk = daily_apks_by_URL(db_result_mapk_today)
IP_result_mapk = daily_apks_by_IP(db_result_mapk_today)
categories_result_APKPure = daily_apks_by_categories(
db_result_APKPure_today)
antivirus_result_APKPure = daily_apks_by_antivirus(db_result_APKPure_today)
permission_result_APKPure = daily_apks_by_permission(
db_result_APKPure_today)
URL_result_APKPure = daily_apks_by_URL(db_result_APKPure_today)
IP_result_APKPure = daily_apks_by_IP(db_result_APKPure_today)
# get data form db
# print db_result_gplay[0]['submit_date']
# print categories_result_gplay
# print categories_result_mapk
#add data for return to html
return render_template('daily_apks.html',
date=today,
categories_gplay=categories_result_gplay,
antivirus_gplay=antivirus_result_gplay,
permission_gplay=permission_result_gplay,
URL_gplay=URL_result_gplay,
IP_gplay=IP_result_gplay,
categories_mapk=categories_result_mapk,
antivirus_mapk=antivirus_result_mapk,
permission_mapk=permission_result_mapk,
URL_mapk=URL_result_mapk,
IP_mapk=IP_result_mapk,
categories_APKPure=categories_result_APKPure,
antivirus_APKPure=antivirus_result_APKPure,
permission_APKPure=permission_result_APKPure,
URL_APKPure=URL_result_APKPure,
IP_APKPure=IP_result_APKPure)
def main():
parser = argparse.ArgumentParser(
description='List subcategories and apps within them.')
parser.add_argument(
"nb_results",
nargs=1,
type=str,
default=None,
help=
'You can get a list of all subcategories available, by supplying a valid category'
)
parser.add_argument(
"--offset",
nargs=1,
type=int,
default=None,
help=
'You can get a list of all subcategories available, by supplying a valid category'
)
args = parser.parse_args()
ctr = "apps_topselling_free"
nb_results = args.nb_results[0]
if args.offset:
logging.debug("there's offset : {}".format(args.offset))
offset = args.offset[0]
else:
offset = None
# Using GooglePlayAPI
api = GooglePlayAPI(ANDROID_ID)
api.login(GOOGLE_LOGIN, GOOGLE_PASSWORD, AUTH_TOKEN)
bar = Bar('Downloading: ', max=34 * int(nb_results))
for cat in category:
try:
message = api.list(cat, ctr, nb_results, offset)
except:
logging.error(
"Error: HTTP 500 - one of the provided parameters is invalid")
raise
# Get info, download apk, calculate file hashes, insert to mongodb
# today = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
today = datetime.now().strftime('%Y-%m-%d')
rank = 1
for i in message.doc[0].child:
result = {'permission': {'p': 'v'}}
result['vt_scan'] = False
result['submit_date'] = today
result['source'] = 'googleplay'
result['title'] = cat
result['sub_title'] = ctr
result['name'] = unicode(i.title).encode('utf8')
result['rank'] = rank
result['pgname'] = i.details.appDetails.packageName
# result['version'] = i.details.appDetails.versionCode
# result['size'] = sizeof_fmt(i.details.appDetails.installationSize)
result['upload_date'] = datetime.strptime(
unicode(i.details.appDetails.uploadDate).encode('utf8'),
locale_timestring[LANG]).strftime('%Y-%m-%d')
# Download APK
result['apkdata'] = api.download(i.details.appDetails.packageName,
i.details.appDetails.versionCode,
i.offer[0].offerType)
# Calculate file hashes
# result.update(File(result['apkdata']).result)
# update rank
rank += 1
# write buffer to apk file
filename = '/tmp/' + result['pgname'] + '.apk'
with open(filename, 'wb') as f:
f.write(result['apkdata'])
# retrive permission from apk file
try:
result.update(Get_info_from_APK(filename, result['apkdata']))
size = float(sys.getsizeof(result['apkdata']))
result['size'] = str(round((size / 1048576), 1)) + 'MB'
print result['sha512']
raw_input('stop')
# result['interesting_strings_URL'] = Find_URL(filename)
# result['interesting_strings_IP'] = Find_IP_address(filename)
except Exception, e:
logging.error("DB insert error: {}".format(result['pgname']))
traceback.print_exc()
# result['danger_permission'], result['normal_permission'] = APK_permission(filename)
# Insert apk into mongodb
try:
DB().insert_apk(result)
bar.next()
except KeyError:
logging.warn("Maybe the apk already exists: {}".format(
result['pgname']))
# continue
raise
except:
logging.error("DB insert error: {}".format(result['pgname']))
raise
def __init__(self,lock,api_key,data):
threading.Thread.__init__(self)
self.v = virustotal.VirusTotal(api_key)
self.db = DB()
self.lock = lock
self.thread_data = data
def Get_apk_name_and_link(category_subpage, apk_category):
result = {}
result['vt_scan'] = False
result['url_scan'] = False
result['source'] = 'androidapksfree.com'
result['title'] = apk_category
result['category'] = []
result['packageName'] = []
result['apkName'] = []
fakeURL_list = []
pkg_name_list = []
dlink_list = []
#Connect to the apk page
apk_category_link = "http://www.androidapksfree.com/applications/apps/" + apk_category
res = Retry_session(apk_category_link, 100)
#Bar_string = 'Processing '+ result['title']
#bar = Bar(Bar_string, max = int(len(pkg_name_list)))
for subpage_link in category_subpage:
subpage = BeautifulSoup(urllib.urlopen('https://' + subpage_link),
"lxml")
subpg = subpage.findAll("div",
{"class": "image-style-for-related-posts"})
for content in xrange(0, len(subpg)):
for url in subpg[content].findAll("a"):
rank = 0
try:
fakeURL = url.get('href')
fakeURL_list.append(fakeURL)
#print "fakeURL : ",fakeURL
apk_name = url.get('href')[36:][:-1]
print colored("downloading", 'red'), apk_name, "..."
pkg_name = _get_pkg_name(fakeURL)
pkg_name_list.append(pkg_name)
dlink = _get_d_link(fakeURL)
dlink_list.append(dlink)
#result['category'] = apk_category
result['pgname'] = pkg_name
#result['apkName'] = apk_name
Get_info_status = False #initialze value
Get_download_status = False #initialze value
Get_info_status = Get_apk_information(dlink, result)
Get_download_status = Download_link(
dlink, apk_name, rank + 1, result)
print "Get_info_status : ", Get_info_status
if Get_info_status is True and Get_download_status is True:
try:
db = DB()
db.insert_apk(result)
except:
traceback.print_exc()
else:
pass
except Exception, e:
traceback.print_exc()