Beispiel #1
0
def save_report(case_name, tags):
    db_store_path = ROOT_DIR + SEP + 'data' + SEP + case_name
    file_path = db_store_path + SEP + 'report' + SEP + 'report.txt'
    mkdir(db_store_path+ SEP + 'report')
    REPORT.append(["Case Name", case_name])
    REPORT.append(["Tags", tags])
    return write_to_file(file_path, REPORT)
Beispiel #2
0
def command_tools():
    tools = 'tools'
    tools_bin = 'tools/bin/%s' % TURBULENZOS

    mkdir(tools_bin)
    if TURBULENZOS == 'win32':
        devenv, vs_version = find_devenv()
        if vs_version == '2008':
            proj_postfix = '.vcproj'
            sln_postfix = '.sln'
        elif vs_version == '2010':
            proj_postfix = '-2010.vcxproj'
            sln_postfix = '-2010.sln'
        elif vs_version == '2012':
            proj_postfix = '-2012.vcxproj'
            sln_postfix = '-2012.sln'

        cgfx2json_proj = os.path.normpath(
                            os.path.join(TURBULENZROOT, tools, 'cgfx2json', 'cgfx2json%s' % proj_postfix))
        sh([devenv, cgfx2json_proj, '/build', 'Release'], console=True)
        cp('tools/cgfx2json/Release/cgfx2json.exe', tools_bin)
        cp('external/Cg/bin/cg.dll', tools_bin)
        cp('external/Cg/bin/cgGL.dll', tools_bin)

        nvtristrip_sln = os.path.normpath(
                            os.path.join(TURBULENZROOT, tools, 'NvTriStrip', 'NvTriStrip%s' % sln_postfix))
        sh([devenv, nvtristrip_sln, '/build', 'Release'], console=True)
        cp('tools/NvTriStrip/NvTriStripper/bin/release/NvTriStripper.exe', tools_bin)

    else:
        sh('make', tools, console=True)
        cp('tools/cgfx2json/bin/release/cgfx2json', tools_bin)
        cp('tools/NvTriStrip/NvTriStripper/bin/release/NvTriStripper', tools_bin)
def train_fold(kfold_df, test_df, params, fold):
    fold_save_path = join(params['save_dir'], 'fold_%d'%fold)
    model_save_path = join(fold_save_path, 'model.pth.tar')
    mkdir(fold_save_path)
    train_df = kfold_df[kfold_df.Fold != fold]
    val_df = kfold_df[(kfold_df.Fold == fold)&(kfold_df.Additional == 0)]

    train_loader = torch.utils.data.DataLoader(
        PandasDataset(train_df, train_transform()),
        batch_size=params['batch_size'], shuffle=True,
        num_workers=2, pin_memory=True)

    val_loader = torch.utils.data.DataLoader(
        PandasDataset(val_df, test_transform()),
        batch_size=params['batch_size'], shuffle=False,
        num_workers=2, pin_memory=True)

    model = get_pretrained_model(params['arch'], params['lr'])
    model.model_path = model_save_path
    history = model.fit(train_loader, val_loader, n_epoch=params['n_epoch'])
    model.load_model(model.model_path)
    val_score = model.validate(val_loader)
    predict_test(model, test_df, fold_save_path)
    save_history(history, fold_save_path)
    del model, train_loader, val_loader
Beispiel #4
0
def store_media_data(session_name):
    global mediadb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    mediadb = OUTPUT + SEP + 'db/external.db'
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_files_data()
Beispiel #5
0
def extract_all_data(session_name):
    global OUTPUT
    if 'root' in PERM:
        OUTPUT = OUTPUT + session_name
        mkdir(OUTPUT)
        mkdir(OUTPUT + SEP + 'db')
        for db in DB_LIST:
            download_database(db)
Beispiel #6
0
def store_bluetooth_data(session_name):
    global bluetoothdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    bluetoothdb = OUTPUT + SEP + 'db/btopp.db'
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_btopp()
Beispiel #7
0
def store_sms_data(session_name):
    global smsdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    smsdb = OUTPUT + SEP + 'db/mmssms.db'
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_sms_messages()
Beispiel #8
0
def start_download_databases(session_name):
    global OUTPUT
    if 'root' in PERM:
        OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
        mkdir(OUTPUT + SEP + 'db')
        for db in DBS_LIST:
            print('Extracting current db from: ' +db)
            download_database(db)
Beispiel #9
0
def store_browser_history_data(session_name):
    global browserdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    browserdb = OUTPUT + SEP + 'db/History'
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_browser_history()
Beispiel #10
0
def command_tools():
    tools = os.path.normpath(os.path.join(TURBULENZROOT, 'tools'))
    tools_bin = os.path.normpath(os.path.join(tools, 'bin', TURBULENZOS))
    mkdir(tools_bin)

    if TURBULENZOS == 'win32':
        devenv, vs_version_name, msbuild = find_devenv()
        if not devenv and not msbuild:
            error('Could not find a valid install of Visual Studio')
            return 1
        if vs_version_name == '2008':
            proj_postfix = '.vcproj'
            sln_postfix = '.sln'
            vs_version = '9.0'
        elif vs_version_name == '2010':
            proj_postfix = '-2010.vcxproj'
            sln_postfix = '-2010.sln'
            vs_version = '10.0'
        elif vs_version_name == '2012':
            proj_postfix = '-2012.vcxproj'
            sln_postfix = '-2012.sln'
            vs_version = '11.0'
        elif vs_version_name == '2013':
            proj_postfix = '-2013.vcxproj'
            sln_postfix = '-2013.sln'
            vs_version = '12.0'
        if devenv:
            base_cmd = [devenv, '/build', 'Release']
        elif msbuild:
            base_cmd = [
                msbuild, '/t:build', '/p:Configuration=Release',
                '/p:Platform=Win32',
                '/p:VisualStudioVersion=%s' % vs_version
            ]

        cgfx2json_proj = os.path.join(tools, 'cgfx2json',
                                      'cgfx2json%s' % proj_postfix)
        cmd = base_cmd + [cgfx2json_proj]
        sh(cmd, console=True, shell=True)
        cp('%s/cgfx2json/Release/cgfx2json.exe' % tools, tools_bin)
        cp('%s/external/Cg/bin/cg.dll' % TURBULENZROOT, tools_bin)
        cp('%s/external/Cg/bin/cgGL.dll' % TURBULENZROOT, tools_bin)

        nvtristrip_sln = os.path.join(tools, 'NvTriStrip',
                                      'NvTriStrip%s' % sln_postfix)
        cmd = base_cmd + [nvtristrip_sln]
        sh(cmd, console=True, shell=True)
        cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper.exe' % tools,
           tools_bin)

    else:
        sh('make', cwd=tools, console=True)
        cp('%s/cgfx2json/bin/release/cgfx2json' % tools, tools_bin)
        cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper' % tools,
           tools_bin)
Beispiel #11
0
def store_wa_data(session_name):
    global msgstoredb
    global contactsdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    msgstoredb = OUTPUT + SEP + "db/msgstore.db"
    contactsdb = OUTPUT + SEP + "db/wa.db"
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_wa_contacts()
    store_wa_messages()
def train_kfolds(params):
    mkdir(params['save_dir'])
    kfold_df, test_df = load_detect_train_test_df(params['data_dir'])
    folds = sorted(kfold_df.Fold.unique())

    with open(join(params['save_dir'], 'params.txt'), 'w') as f:
        f.write(str(params))
    
    for fold in folds:
        print("Start train fold %d" % fold)
        train_fold(kfold_df, test_df, params, fold)
Beispiel #13
0
def store_location_data(session_name):
    global savedlocationdb
    global searchlocationdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    savedlocationdb = OUTPUT + SEP + 'db/gmm_myplaces.db'
    searchlocationdb = OUTPUT + SEP + 'db/gmm_storage.db'
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_saved_location()
    store_searched_location()
Beispiel #14
0
def store_fb_data(session_name):
    global threadsdb
    global contactsdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    threadsdb = OUTPUT + SEP + "db/threads_db2"
    contactsdb = OUTPUT + SEP + "db/contacts_db2"
    OUTPUT = OUTPUT + SEP + 'tsv'
    print("folder = " + threadsdb)
    mkdir(OUTPUT)
    store_fb_contacts()
    store_fb_thread_data()
Beispiel #15
0
def store_phone_data(session_name):
    global msgsdb
    global contactsdb
    global calllogsdb
    global OUTPUT
    OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
    msgsdb = OUTPUT + SEP + "db/bugle_db"
    contactsdb = OUTPUT + SEP + "db/dialer.db"
    calllogsdb = OUTPUT + SEP + "db/contacts2.db"
    OUTPUT = OUTPUT + SEP + 'tsv'
    mkdir(OUTPUT)
    store_call_logs()
    store_phone_contacts()
    store_text_messages()
Beispiel #16
0
def command_tools():
    tools = os.path.normpath(os.path.join(TURBULENZROOT, 'tools'))
    tools_bin = os.path.normpath(os.path.join(tools, 'bin', TURBULENZOS))
    mkdir(tools_bin)

    if TURBULENZOS == 'win32':
        devenv, vs_version_name, msbuild = find_devenv()
        if not devenv and not msbuild:
            error('Could not find a valid install of Visual Studio')
            return 1
        if vs_version_name == '2008':
            proj_postfix = '.vcproj'
            sln_postfix = '.sln'
            vs_version = '9.0'
        elif vs_version_name == '2010':
            proj_postfix = '-2010.vcxproj'
            sln_postfix = '-2010.sln'
            vs_version = '10.0'
        elif vs_version_name == '2012':
            proj_postfix = '-2012.vcxproj'
            sln_postfix = '-2012.sln'
            vs_version = '11.0'
        elif vs_version_name == '2013':
            proj_postfix = '-2013.vcxproj'
            sln_postfix = '-2013.sln'
            vs_version = '12.0'
        if devenv:
            base_cmd = [devenv, '/build', 'Release']
        elif msbuild:
            base_cmd = [msbuild, '/t:build', '/p:Configuration=Release',
                        '/p:Platform=Win32', '/p:VisualStudioVersion=%s' % vs_version]

        cgfx2json_proj = os.path.join(tools, 'cgfx2json', 'cgfx2json%s' % proj_postfix)
        cmd = base_cmd + [cgfx2json_proj]
        sh(cmd, console=True, shell=True)
        cp('%s/cgfx2json/Release/cgfx2json.exe' % tools, tools_bin)
        cp('%s/external/Cg/bin/cg.dll' % TURBULENZROOT, tools_bin)
        cp('%s/external/Cg/bin/cgGL.dll' % TURBULENZROOT, tools_bin)

        nvtristrip_sln = os.path.join(tools, 'NvTriStrip', 'NvTriStrip%s' % sln_postfix)
        cmd = base_cmd + [nvtristrip_sln]
        sh(cmd, console=True, shell=True)
        cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper.exe' % tools, tools_bin)

    else:
        sh('make', cwd=tools, console=True)
        cp('%s/cgfx2json/bin/release/cgfx2json' % tools, tools_bin)
        cp('%s/NvTriStrip/NvTriStripper/bin/release/NvTriStripper' % tools, tools_bin)
Beispiel #17
0
def extract_all_data_toTsv(session_name):
    global OUTPUT
    if 'root' in PERM:
        OUTPUT = OUTPUT + SEP + 'data' + SEP + session_name
        mkdir(OUTPUT + SEP + 'db')
        file_dir = file_dir = ROOT_DIR + '/data/' + session_name + '/tsv/'
        src_path = ROOT_DIR + '/data/' + session_name + '/db/'
        mkdir(file_dir)
        for db in DB_LIST:
            print('Extracting current db from: ' + db)
            download_database(db)
            db_name = db.split('/')[-1]
            try:
                convert_to_tsv(src_path + db_name, file_dir)
            except:
                print("unable to convert " + db)
Beispiel #18
0
def save_report(session_name):
    db_store_path = ROOT_DIR + SEP + 'data' + SEP + session_name
    file_path = db_store_path + SEP + 'report' + SEP + 'report.txt'
    mkdir(db_store_path + SEP + 'report')
    REPORT.append(["Case Name", session_name])
    return write_to_file(file_path, REPORT)