def save_report(case_name, tags):
db_store_path = ROOT_DIR + SEP + 'data' + SEP + case_name
file_path = db_store_path + SEP + 'report' + SEP + 'report.txt'
mkdir(db_store_path+ SEP + 'report')
REPORT.append(["Case Name", case_name])
REPORT.append(["Tags", tags])
return write_to_file(file_path, REPORT)
def command_tools():
tools = 'tools'
tools_bin = 'tools/bin/%s' % TURBULENZOS
mkdir(tools_bin)
if TURBULENZOS == 'win32':
devenv, vs_version = find_devenv()
if vs_version == '2008':
proj_postfix = '.vcproj'
sln_postfix = '.sln'
elif vs_version == '2010':
proj_postfix = '-2010.vcxproj'
sln_postfix = '-2010.sln'
elif vs_version == '2012':
proj_postfix = '-2012.vcxproj'
sln_postfix = '-2012.sln'
cgfx2json_proj = os.path.normpath(
os.path.join(TURBULENZROOT, tools, 'cgfx2json', 'cgfx2json%s' % proj_postfix))
sh([devenv, cgfx2json_proj, '/build', 'Release'], console=True)
cp('tools/cgfx2json/Release/cgfx2json.exe', tools_bin)
cp('external/Cg/bin/cg.dll', tools_bin)
cp('external/Cg/bin/cgGL.dll', tools_bin)
nvtristrip_sln = os.path.normpath(
os.path.join(TURBULENZROOT, tools, 'NvTriStrip', 'NvTriStrip%s' % sln_postfix))
sh([devenv, nvtristrip_sln, '/build', 'Release'], console=True)
cp('tools/NvTriStrip/NvTriStripper/bin/release/NvTriStripper.exe', tools_bin)
else:
sh('make', tools, console=True)
cp('tools/cgfx2json/bin/release/cgfx2json', tools_bin)
cp('tools/NvTriStrip/NvTriStripper/bin/release/NvTriStripper', tools_bin)
def train_fold(kfold_df, test_df, params, fold):
fold_save_path = join(params['save_dir'], 'fold_%d'%fold)
model_save_path = join(fold_save_path, 'model.pth.tar')
mkdir(fold_save_path)
train_df = kfold_df[kfold_df.Fold != fold]
val_df = kfold_df[(kfold_df.Fold == fold)&(kfold_df.Additional == 0)]
train_loader = torch.utils.data.DataLoader(
PandasDataset(train_df, train_transform()),
batch_size=params['batch_size'], shuffle=True,
num_workers=2, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
PandasDataset(val_df, test_transform()),
batch_size=params['batch_size'], shuffle=False,
num_workers=2, pin_memory=True)
model = get_pretrained_model(params['arch'], params['lr'])
model.model_path = model_save_path
history = model.fit(train_loader, val_loader, n_epoch=params['n_epoch'])
model.load_model(model.model_path)
val_score = model.validate(val_loader)
predict_test(model, test_df, fold_save_path)
save_history(history, fold_save_path)
del model, train_loader, val_loader
def store_media_data(session_name):
global mediadb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
mediadb = OUTPUT + SEP + 'db/external.db'
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_files_data()
def extract_all_data(session_name):
global OUTPUT
if 'root' in PERM:
OUTPUT = OUTPUT + session_name
mkdir(OUTPUT)
mkdir(OUTPUT + SEP + 'db')
for db in DB_LIST:
download_database(db)
def store_bluetooth_data(session_name):
global bluetoothdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
bluetoothdb = OUTPUT + SEP + 'db/btopp.db'
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_btopp()
def store_sms_data(session_name):
global smsdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
smsdb = OUTPUT + SEP + 'db/mmssms.db'
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_sms_messages()
def start_download_databases(session_name):
global OUTPUT
if 'root' in PERM:
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
mkdir(OUTPUT + SEP + 'db')
for db in DBS_LIST:
print('Extracting current db from: ' +db)
download_database(db)
def store_browser_history_data(session_name):
global browserdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
browserdb = OUTPUT + SEP + 'db/History'
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_browser_history()
def command_tools():
tools = os.path.normpath(os.path.join(TURBULENZROOT, 'tools'))
tools_bin = os.path.normpath(os.path.join(tools, 'bin', TURBULENZOS))
mkdir(tools_bin)
if TURBULENZOS == 'win32':
devenv, vs_version_name, msbuild = find_devenv()
if not devenv and not msbuild:
error('Could not find a valid install of Visual Studio')
return 1
if vs_version_name == '2008':
proj_postfix = '.vcproj'
sln_postfix = '.sln'
vs_version = '9.0'
elif vs_version_name == '2010':
proj_postfix = '-2010.vcxproj'
sln_postfix = '-2010.sln'
vs_version = '10.0'
elif vs_version_name == '2012':
proj_postfix = '-2012.vcxproj'
sln_postfix = '-2012.sln'
vs_version = '11.0'
elif vs_version_name == '2013':
proj_postfix = '-2013.vcxproj'
sln_postfix = '-2013.sln'
vs_version = '12.0'
if devenv:
base_cmd = [devenv, '/build', 'Release']
elif msbuild:
base_cmd = [
msbuild, '/t:build', '/p:Configuration=Release',
'/p:Platform=Win32',
'/p:VisualStudioVersion=%s' % vs_version
]
cgfx2json_proj = os.path.join(tools, 'cgfx2json',
'cgfx2json%s' % proj_postfix)
cmd = base_cmd + [cgfx2json_proj]
sh(cmd, console=True, shell=True)
cp('%s/cgfx2json/Release/cgfx2json.exe' % tools, tools_bin)
cp('%s/external/Cg/bin/cg.dll' % TURBULENZROOT, tools_bin)
cp('%s/external/Cg/bin/cgGL.dll' % TURBULENZROOT, tools_bin)
nvtristrip_sln = os.path.join(tools, 'NvTriStrip',
'NvTriStrip%s' % sln_postfix)
cmd = base_cmd + [nvtristrip_sln]
sh(cmd, console=True, shell=True)
cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper.exe' % tools,
tools_bin)
else:
sh('make', cwd=tools, console=True)
cp('%s/cgfx2json/bin/release/cgfx2json' % tools, tools_bin)
cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper' % tools,
tools_bin)
def store_wa_data(session_name):
global msgstoredb
global contactsdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
msgstoredb = OUTPUT + SEP + "db/msgstore.db"
contactsdb = OUTPUT + SEP + "db/wa.db"
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_wa_contacts()
store_wa_messages()
def train_kfolds(params):
mkdir(params['save_dir'])
kfold_df, test_df = load_detect_train_test_df(params['data_dir'])
folds = sorted(kfold_df.Fold.unique())
with open(join(params['save_dir'], 'params.txt'), 'w') as f:
f.write(str(params))
for fold in folds:
print("Start train fold %d" % fold)
train_fold(kfold_df, test_df, params, fold)
def store_location_data(session_name):
global savedlocationdb
global searchlocationdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
savedlocationdb = OUTPUT + SEP + 'db/gmm_myplaces.db'
searchlocationdb = OUTPUT + SEP + 'db/gmm_storage.db'
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_saved_location()
store_searched_location()
def store_fb_data(session_name):
global threadsdb
global contactsdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
threadsdb = OUTPUT + SEP + "db/threads_db2"
contactsdb = OUTPUT + SEP + "db/contacts_db2"
OUTPUT = OUTPUT + SEP + 'tsv'
print("folder = " + threadsdb)
mkdir(OUTPUT)
store_fb_contacts()
store_fb_thread_data()
def store_phone_data(session_name):
global msgsdb
global contactsdb
global calllogsdb
global OUTPUT
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
msgsdb = OUTPUT + SEP + "db/bugle_db"
contactsdb = OUTPUT + SEP + "db/dialer.db"
calllogsdb = OUTPUT + SEP + "db/contacts2.db"
OUTPUT = OUTPUT + SEP + 'tsv'
mkdir(OUTPUT)
store_call_logs()
store_phone_contacts()
store_text_messages()
def command_tools():
tools = os.path.normpath(os.path.join(TURBULENZROOT, 'tools'))
tools_bin = os.path.normpath(os.path.join(tools, 'bin', TURBULENZOS))
mkdir(tools_bin)
if TURBULENZOS == 'win32':
devenv, vs_version_name, msbuild = find_devenv()
if not devenv and not msbuild:
error('Could not find a valid install of Visual Studio')
return 1
if vs_version_name == '2008':
proj_postfix = '.vcproj'
sln_postfix = '.sln'
vs_version = '9.0'
elif vs_version_name == '2010':
proj_postfix = '-2010.vcxproj'
sln_postfix = '-2010.sln'
vs_version = '10.0'
elif vs_version_name == '2012':
proj_postfix = '-2012.vcxproj'
sln_postfix = '-2012.sln'
vs_version = '11.0'
elif vs_version_name == '2013':
proj_postfix = '-2013.vcxproj'
sln_postfix = '-2013.sln'
vs_version = '12.0'
if devenv:
base_cmd = [devenv, '/build', 'Release']
elif msbuild:
base_cmd = [msbuild, '/t:build', '/p:Configuration=Release',
'/p:Platform=Win32', '/p:VisualStudioVersion=%s' % vs_version]
cgfx2json_proj = os.path.join(tools, 'cgfx2json', 'cgfx2json%s' % proj_postfix)
cmd = base_cmd + [cgfx2json_proj]
sh(cmd, console=True, shell=True)
cp('%s/cgfx2json/Release/cgfx2json.exe' % tools, tools_bin)
cp('%s/external/Cg/bin/cg.dll' % TURBULENZROOT, tools_bin)
cp('%s/external/Cg/bin/cgGL.dll' % TURBULENZROOT, tools_bin)
nvtristrip_sln = os.path.join(tools, 'NvTriStrip', 'NvTriStrip%s' % sln_postfix)
cmd = base_cmd + [nvtristrip_sln]
sh(cmd, console=True, shell=True)
cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper.exe' % tools, tools_bin)
else:
sh('make', cwd=tools, console=True)
cp('%s/cgfx2json/bin/release/cgfx2json' % tools, tools_bin)
cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper' % tools, tools_bin)
def extract_all_data_toTsv(session_name):
global OUTPUT
if 'root' in PERM:
OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
mkdir(OUTPUT + SEP + 'db')
file_dir = file_dir = ROOT_DIR + '/data/' + session_name + '/tsv/'
src_path = ROOT_DIR + '/data/' + session_name + '/db/'
mkdir(file_dir)
for db in DB_LIST:
print('Extracting current db from: ' + db)
download_database(db)
db_name = db.split('/')[-1]
try:
convert_to_tsv(src_path + db_name, file_dir)
except:
print("unable to convert " + db)
def save_report(session_name):
db_store_path = ROOT_DIR + SEP + 'data' + SEP + session_name
file_path = db_store_path + SEP + 'report' + SEP + 'report.txt'
mkdir(db_store_path + SEP + 'report')
REPORT.append(["Case Name", session_name])
return write_to_file(file_path, REPORT)