def __init__(self, apk, pbar=1):
"""
Input AndroidSampleModel
"""
self.apk = apk
self.db = DataBase()
self.pbar = pbar
class ApiGenerator:
def __init__(self, apis):
self.apis = apis
self.db = DataBase()
# The list of all api id in current apk
self.api_pool = []
def initialize(self):
for api in tqdm(self.apis, desc="Initialize APIs", leave=False):
api_model = APIModel(api)
self.db.insert_api(api_model)
yield api_model
def generate(self):
"""
generate apis and insert to database
"""
for api in self.apis:
api_model = APIModel(api)
yield api_model
def initDb(config: AppConfig, context: Context):
uri = config.db.url #r'sqlite///db.sqlite'
engin = create_engine(uri, pool_pre_ping=True)
db = DataBase(connection=engin)
db.checkConnect()
context.set('db', db)
def init_db_postgres(config: ApplicationConfig, context: Context):
engine = create_engine(
config.database.url,
pool_pre_ping=True,
)
database = DataBase(connection=engine)
database.check_connection()
context.set('database', database)
def init_db_sqlite(config: ApplicationConfig, context: Context):
engine = create_engine(
name_or_url=config.database.url,
pool_pre_ping=True, # if session dropped, it'll automatically launched
)
database = DataBase(connection=engine)
database.check_connection()
context.set('database', database)
def init_db_sqlite(config: ApplicationConfig, context: Context):
engine = create_engine(
config.database.url,
pool_pre_ping=True, # для переподключения
)
database = DataBase(connection=engine)
database.check_connection()
context.set('database', database)
def init_db(config: ApplicationConfig, context: Context):
"""
Инициализирует подключение к БД и передает его приложению.
"""
engine = create_engine(
config.database.url,
pool_pre_ping=True, # автоматическое восстановление подключения к БД
)
database = DataBase(connection=engine)
database.check_connection()
context.set('database', database)
def __init__(self, apk):
if not self.parse_apk(apk):
self.parsable = False
print(self.obj)
self.db = DataBase()
self.apk = apk
self.apk_hash = tools.sha256sum(apk)
self.parsable = True
self.db.create_sample_data(self.obj)
def __init__(self, **kwargs):
super(MasterDateApp, self).__init__(**kwargs)
global fb
global db
fb = FireBase(self)
db = DataBase(self)
# user num for database entries
self.user_num = self.get_user_num()
def __init__(self):
global db_instance
db_instance = None
if db_instance == None:
db_instance = DataBase.instance()
# 建立和数据库系统的连接
self.conn = db_instance.get_db()
#获取操作游标
self.cursor = self.conn.cursor(pymysql.cursors.DictCursor)
def init_sqlite_db(context: Context, config: ApplicationConfig):
"""
:param context: Mutable
"""
engine = create_engine(f'sqlite:///db.sqlite', pool_pre_ping=True)
engine.execute('pragma foreign_keys=on')
database = DataBase(engine)
# database.check_connection()
context.set("database", database)
def __init__(self, **kwargs):
super(MasterDateApp, self).__init__(**kwargs)
self.fb = FireBase(self)
self.db = DataBase(self)
self.date_keyboard = DateKeyboard()
# user num for database entries
self.store_file = JsonStore(join(self.user_data_dir, 'store_file.txt'))
try:
self.storage = self.store_file.get('data')['content']
self.user = self.storage['user_data']
self.user_num = self.user['num']
except KeyError:
self.storage = {}
self.user = {
'num': str(random.randint(1000000000, 9999999999)),
'level': 1,
'skip_tutorial': 0
}
self.storage['user_data'] = self.user
self.user_num = self.user['num']
self.save()
class ChosunRawDao:
def __init__(self):
self.db = DataBase()
def saveMeta(self, year, month, total, page, page_item):
self.db.insert("""
INSERT INTO chosun_meta (
year, month, total_count, page, items
) VALUES (
%s, %s, %s, %s, %s
)
""", (year, month, total, page, page_item))
def saveRaw(self, year, month, page, params):
def dict_to_tuple(x):
return (year, month, page, x.get("title"), x.get("link"),
x.get("date"), x.get("content_abbr"))
binding = [dict_to_tuple(i) for i in params]
self.db.insert_many("""
INSERT INTO chosun_raw (
year, month, page, title, link,
press_date, content_abbr
) VALUES (
%s, %s, %s, %s, %s,
%s, %s
)
""", binding)
def findRaw(self, year, month):
return self.db.queryList("""
SELECT id, year, month, page, title, link,
press_date
FROM chosun_raw
WHERE year = %s
AND month = %s
AND content is null
AND is_photo = false
AND no_content = false
AND data_error = false
""", (year, month))
def updateRaws(self, content_id_tuples):
print(content_id_tuples)
print(f"updating {len(content_id_tuples)} rows")
self.db.update_many("""
UPDATE chosun_raw
SET content = %s
WHERE id = %s
""", content_id_tuples)
class FetchRawDao:
def __init__(self):
self.db = DataBase()
def queryBigKinds(self, year = None, page = None):
query = """
SELECT
press_date AS date, title, content
FROM bigkind_raw
WHERE content is not null
"""
if year != None:
query = query + f" AND year = '{year}'"
if page != None:
query = query + f" AND page = '{page}'"
return self.db.queryList(query,())
def queryChosun(self, year = None, page = None):
query = """
SELECT
press_date AS date, title, content
FROM chosun_archive
WHERE content is not null AND length(content) > 20
"""
if year != None:
query = query + f" AND year = '{year}'"
if page != None:
query = query + f" AND page = '{page}'"
return self.db.queryList(query,())
class ChosunArchiveDao:
def __init__(self):
self.db = DataBase()
def findRaw(self, year, page):
query = """
SELECT id, article_id, year, page, title, link,
press_date
FROM chosun_archive
WHERE year = %s
AND (content is null or length(content) < 20)
AND is_photo = false
AND no_content = false
AND data_error = false
AND article_id is not null
"""
if page != None:
query = query + f" AND page = '{page}'"
return self.db.queryList(query, (year))
def __init__(self, apis):
self.apis = apis
self.db = DataBase()
# The list of all api id in current apk
self.api_pool = []
from pymongo import MongoClient
from multiprocessing import Pool, Process, Event
from tqdm import tqdm
from model.android_sample_model import AndroidSampleModel
from generator.method_generator import MethodCombGenerator
from generator.api_generator import ApiGenerator
from quark.Objects.quark import Quark
from db.database import DataBase
from utils.tools import distribute, api_filter
from itertools import repeat
db = DataBase()
@click.command()
@click.option(
"-a",
"--apk",
help="APK file",
type=click.Path(exists=True, file_okay=True, dir_okay=False),
required=True,
)
@click.option("-e",
"--export",
help="Export all rules of apk to JSON file",
type=click.Path(exists=True, file_okay=False, dir_okay=True))
@click.option(
"Sargento Cheese Shredded, Traditional Cut, Extra Sharp Cheddar, 7.0 oz",
"ftype": "oos",
"img": "https://docs.google.com/uc?id=",
"price": "$0.00",
"upc": upc
})
id_token = ""
if __name__ == '__main__':
class Blank():
local_id = ""
user_num = "5404136933"
selected_dep = "Dairy"
obj = Blank()
db = DataBase(obj)
date = "10/21/2019"
upc = "2100007129"
sect = "050101010"
a = mark(date, upc)
b = (sect, a)
c = db.post_item(b)
print(c.result)
class MasterDateApp(App):
todo_list = None
def __init__(self, **kwargs):
super(MasterDateApp, self).__init__(**kwargs)
self.fb = FireBase(self)
self.db = DataBase(self)
self.date_keyboard = DateKeyboard()
# user num for database entries
self.store_file = JsonStore(join(self.user_data_dir, 'store_file.txt'))
try:
self.storage = self.store_file.get('data')['content']
self.user = self.storage['user_data']
self.user_num = self.user['num']
except KeyError:
self.storage = {}
self.user = {
'num': str(random.randint(1000000000, 9999999999)),
'level': 1,
'skip_tutorial': 0
}
self.storage['user_data'] = self.user
self.user_num = self.user['num']
self.save()
def save(self):
self.store_file.put('data', content=self.storage)
def on_stop(self):
self.save()
def build(self):
print('user_num', self.user_num)
# Loader.loading_image = 'images/loading.gif'
try:
filename = join(self.user_data_dir, "refresh_token.txt")
with open(filename, 'r') as f:
refresh_token = f.read()
except FileNotFoundError:
# Open login page
sm.add_widget(Login(name='login'))
return sm
else:
# if refresh_token and data
req = self.fb.exchange_refresh_token(refresh_token)
if req['success']:
self.id_token = req['id_token']
self.local_id = req['local_id']
self.login_attempts = 0
return self.login()
else:
# return login
login_screen = Login(name='login')
login_screen.error_msg.text = 'Expired Token(Login or try again)'
login_screen.add_widget(login_screen.try_btn)
sm.add_widget(login_screen)
return sm
def login(self, *args):
# return main
req = self.fb.get_data()
print('app.login')
if args and req['success']:
# if args, login was called from the login screen
sm.add_widget(DepSelect(name='dep_select'))
sm.current = 'dep_select'
elif req['success']:
sm.add_widget(DepSelect(name='dep_select'))
return sm
else:
if req['error'] == 'call_todo':
req = self.db.call_todo(self.id_token, self.local_id)
if req['success']:
print('todo was success')
if args:
return self.login(1)
else:
return self.login()
# remove this
elif self.login_attempts <= 3:
print('trying login again')
self.login_attempts += 1
if args:
return self.login(1)
else:
return self.login()
else:
# return a blank screen, open trouble popup, with try again
# as partial(self.login, 1)
raise Exception('connection')
# remove this
elif req['error'] == 'request' and self.login_attempts <= 3:
print('trying login again')
self.login_attempts += 1
if args:
return self.login(1)
else:
return self.login()
else:
# return a blank screen, open trouble popup, with try again
# as partial(self.login, 1)
raise Exception('connection')
def on_start(self):
from kivy.base import EventLoop
EventLoop.window.bind(on_keyboard=self.hook_keyboard)
def hook_keyboard(self, window, key, *largs):
if key == 27:
sm.current_screen.back_pressed()
# self.stop()
return True
def on_pause(self):
self.save()
return True
def on_resume(self):
# Check connection to DB
return True
class MethodCombGenerator:
def __init__(self, apk, pbar=1):
"""
Input AndroidSampleModel
"""
self.apk = apk
self.db = DataBase()
self.pbar = pbar
def check_apk_parsable(self):
"""
Check if given apk are parsable
"""
if not self.apk.parsable:
print(self.apk.obj)
return False
return True
def parse_apk(self):
"""
Parse apk by androguard.
:return: failed or succeed
"""
try:
self.apk_analysis = Quark(self.apk)
except:
return False
return True
def get_permissions(self):
"""
Return all native permissions usage from apk.
:return: a set() for all native permissions usage
"""
perm = set(self.apk_analysis.apkinfo.permissions)
for p in perm.copy():
if not "android.permission" in p:
perm.discard(p)
return perm
def set_progress_status(self, status):
if not self.db.set_status(self.apk.id, status):
print("There is some mistake while setting progress, recheck")
if not self.check_progress:
return False
return True
def check_progress(self):
apk_status = self.db.get_progress_status(self.apk.id)
if not apk_status:
print(colors.yellow("wait for a second then check"))
return False
if apk_status == 1:
print("{} as known as {} has done analysis!".format(
colors.yellow(self.apk.name), self.apk.id))
return False
elif apk_status == 4:
print("{} as known as {} is failed parsing!".format(
colors.yellow(self.apk.name), self.apk.id))
return False
return True
def first_stage_rule_generate(self, apis_pool):
"""
Extract all api usage in apk current apk then generate method combination.
"""
# Check if apk parsable
if not self.check_apk_parsable():
self.set_progress_status(4)
return
# Check apk progress
if not self.check_progress():
return
api_generator = ApiGenerator(self.apk.apis)
origin_apis = list(api_generator.generate())
# Setup progress bar
origin_apis_num = len(origin_apis)
outter_desc = f"Core No.{self.pbar}"
outter_loop = tqdm(apis_pool,
desc=outter_desc,
position=self.pbar,
leave=False)
for api1 in apis_pool:
outter_loop.update(1)
# Tag the method
if not self.db.check_analysis_progress(api1.id, self.apk.id):
tqdm.write(
"{}, {}->{} has done in apk progress move forward".format(
colors.lightblue(api1.id), api1.class_name,
api1.method_name))
continue
# Skip current api if it is not exist
if self.apk.apk_analysis.apkinfo.find_method(
api1.class_name, api1.method_name,
api1.descriptor) is None:
tqdm.write(
"{}, {}->{} does not exist in apk move forward".format(
api1.id, api1.class_name, api1.method_name))
continue
matched_list = []
id_list = []
for num, api2 in enumerate(origin_apis, start=1):
inner_desc = f"{num}/{origin_apis_num}"
outter_loop.set_postfix(inner_loop=inner_desc, refresh=True)
api = api2
# Skip same api
if api2.id == api1.id:
continue
_comb = {
"crime":
"",
"x1_permission": [],
"x2n3n4_comb": [{
"class": api1.class_name,
"method": api1.method_name,
"descriptor": api1.descriptor
}, {
"class": api2.class_name,
"method": api2.method_name,
"descriptor": api2.descriptor
}],
"yscore":
1
}
comb = GenRuleObject(_comb)
try:
result = self.apk.apk_analysis.run(comb)
except KeyboardInterrupt:
self.set_progress_status(2)
raise
except Exception as e:
tqdm.write(
"{} and {} combination has some error when analyzing, ERROR MESSAGE: {}"
.format(api1.id, api2.id, e))
continue
if not comb.check_item[4]:
continue
# tqdm.write("{} and {} matched check!".format(
# colors.green(api1.id), colors.green(api2.id)))
matched_list.append({"m1": api1.id, "m2": api2.id})
id_list.append(api1.id + api2.id)
# Insert matched combination to database and update progress
if not self.db.update_analysis_progress(api1.id, self.apk.id):
tqdm.write(f"Error occured while update progress: {api1.id}")
if not self.db.save_matched_comb(self.apk.id, matched_list,
id_list):
tqdm.write("some Error occure")
# Second stage rule generate
# self.sec_stage_rule_generate(matched_list)
# Apk completed analyzing
outter_loop.clear()
outter_loop.close()
self.set_progress_status(1)
return
class AndroidSampleModel:
def __init__(self, apk):
if not self.parse_apk(apk):
self.parsable = False
print(self.obj)
self.db = DataBase()
self.apk = apk
self.apk_hash = tools.sha256sum(apk)
self.parsable = True
self.db.create_sample_data(self.obj)
def parse_apk(self, apk):
"""
Parse apk by androguard.
:return: failed or succeed
"""
try:
self.apk_analysis = Quark(apk)
except:
return False
return True
@property
def id(self):
"""
Return apk_hash
:return: a string of hashcode sha216
"""
return self.apk_hash
@property
def name(self):
"""
Return apk filename
:return: a string of apk filename
"""
return self.apk_analysis.apkinfo.filename
@property
def permissions(self):
"""
Return a list of permissions usage
:return: a list of permissions
"""
if not self.parsable:
return None
perm = set(self.apk_analysis.apkinfo.permissions)
for p in perm.copy():
if not "android.permission" in p:
perm.discard(p)
return list(perm)
@property
def apis(self):
"""
Return a list of APIModel
:return: a list that consist with APIModel
"""
if not self.parsable:
return None
result = list()
for cls in self.apk_analysis.apkinfo.analysis.get_external_classes():
for meth_analysis in cls.get_methods():
if meth_analysis.is_android_api():
result.append(meth_analysis)
return result
@property
def status(self):
if not self.parsable:
return None
return 5
@property
def report(self):
if not self.parsable:
return None
return
@property
def obj(self):
"""
The object data for database
"""
_obj = {
"_id": self.id,
"hash": self.id,
"filename": self.name,
"parsable": self.parsable,
"permissions": self.permissions,
"status": self.status,
"report": None,
"api_num": len(self.apis),
"progress": []
}
return _obj
def __init__(self): self.db = DataBase()