Ejemplo n.º 1
0
def fetch(request):
    firmware_id = req_get_param(request, 'firmware_id')
    if StrUtils.is_blank(firmware_id):
        return sys_app_err('ERROR_INVALID_PARAMETER')
    doc = firmware_db.fetch(firmware_id)
    if doc is None:
        #SysLog.fail('提取漏洞', '没有提取到漏洞信息(ID={})'.format(firmware_id))
        return sys_app_err('ERROR_FWID_NOT_FOUND')
    #SysLog.success('提取漏洞', '成功提取漏洞信息(ID={})'.format(firmware_id))
    return app_ok_p(doc)
Ejemplo n.º 2
0
def angr_recognize_func(request):
    functions = []
    try:
        filename = req_get_param(request, 'filename')
        arch = getarch(filename)
        proj = angr.Project(filename,
                            load_options={
                                'auto_load_libs': False,
                                'main_opts': {
                                    'backend': 'blob',
                                    'base_addr': 0,
                                    'arch': arch,
                                },
                            })
        cfg = proj.analyses.CFGFast()
        for address, func in cfg.functions.items():
            print(hex(address), func.name)
            functions.append(func.name)

    except binwalk.ModuleException as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({
        'functions': functions,
    })
Ejemplo n.º 3
0
def binwalk_scan_opcodes(request):
    filename = req_get_param(request, 'filename')
    #print(filename)
    # filename = "D:/code/work/firmwareanalyze/HC5611.bin"
    structure = ''
    try:
        for module in binwalk.scan(filename, opcodes=True, quiet=True):
            print("%s Results:" % module.name)
            for result in module.results:
                print("\t%s    0x%.8X    %s" %
                      (result.file.path, result.offset, result.description))
                if ("X86" in result.description.upper()):
                    structure = 'X86'
                    break
                elif ("ARM" in result.description.upper()):
                    structure = "ARM"
                    break
                elif ("MIPS" in result.description.upper()):
                    structure = "MIPS"
                    break
                else:
                    structure = "PowerPC"
                    break
    except binwalk.ModuleException as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({
        'structure': structure,
    })
Ejemplo n.º 4
0
def binwalk_file_test(request):
    filename = req_get_param(request, 'filename')
    try:

        for module in binwalk.scan(filename, filesystem=True, quiet=True):
            for result in module.results:
                if result.file.path in module.extractor.output:
                    # These are files that binwalk carved out of the original firmware image, a la dd
                    if result.offset in module.extractor.output[
                            result.file.path].carved:
                        print
                        "Carved data from offset 0x%X to %s" % (
                            result.offset, module.extractor.output[
                                result.file.path].carved[result.offset])
                    # These are files/directories created by extraction utilities (gunzip, tar, unsquashfs, etc)
                    if result.offset in module.extractor.output[
                            result.file.path].extracted:
                        print
                        "Extracted %d files from offset 0x%X to '%s' using '%s'" % (
                            len(module.extractor.output[result.file.path].
                                extracted[result.offset].files), result.offset,
                            module.extractor.output[result.file.path].
                            extracted[result.offset].files[0],
                            module.extractor.output[result.file.path].
                            extracted[result.offset].command)

    except binwalk.ModuleException as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({
        'extract': 'ok',
    })
Ejemplo n.º 5
0
    def cosine_algorithm(self, fw_file_id, component_file_id):

        # 1 从存储桶导出相关文件
        fw_file_path = FwFilesStorage.export(fw_file_id)
        if fw_file_path is None:
            return sys_app_err('ERROR_INVALID_PARAMETER')

        component_file_path = MakeCOMFilesStorage.export(component_file_id)
        if component_file_path is None:
            return sys_app_err('ERROR_INVALID_PARAMETER')

        # 两篇待比较的文档的路径
        # sourcefile = 'E:/samples/11.txt'
        # s2 = 'E:/samples/22.txt'
        # sourcefile = 'E:/samples/argv_test'
        # s2 = 'E:/samples/argv_test1'

        T1 = Assembly.Count(fw_file_path)
        # print("文档1的词频统计如下:")
        # print(T1)
        T2 = Assembly.Count(component_file_path)
        # print("文档2的词频统计如下:")
        # print(T2)
        # 合并两篇文档的关键词
        mergeword = Assembly.MergeWord(T1, T2)
        # print(mergeword)
        # print(len(mergeword))
        # 得出文档向量
        v1 = Assembly.CalVector(T1, mergeword)
        # print("文档1向量化得到的向量如下:")
        # print(v1)
        v2 = Assembly.CalVector(T2, mergeword)
        # print("文档2向量化得到的向量如下:")
        # print(v2)
        # 计算余弦距离
        # cosine_percent = Assembly.CalConDis(v1, v2, len(v1)) + '%'
        cosine_percent = format(Assembly.CalConDis(v1, v2, len(v1)), ".2f") + '%'

        return sys_app_ok_p({'cosine_percent': cosine_percent})
def taint_func_list(request):
    # 从请求中取参数:文件 ID
    file_id = ReqParams.one(request, 'file_id')

    # 查找函数列表分析结果
    # 查询文件 CFG 分析的标记
    is_cfg = CfgAnalyzeService.has_cfg_analyze(file_id)
    if not is_cfg:
        # 启动分析任务
        task_id = CfgAnalyzeService.start_cfg_task(file_id)
        # 保存操作日志
        LogRecords.save({
            'task_id': task_id,
            'file_id': file_id
        },
                        category='analysis',
                        action='分析CFG',
                        desc='对二进制文件做调用流程图分析')

        # 返回响应:任务初始化的信息
        return sys_app_ok_p(MyTask.fetch_exec_info(task_id))

    # 启动分析任务
    functions = FilesService.functions_list(file_id)
    if len(functions) == 0:
        return sys_app_err(Error.FW_FILE_NO_CFG_ANALYZE)

    taint_func_list = []

    taint_funcs = []

    taint_list = func_taint_col.find()

    for taint_info in taint_list:
        taint_funcs.append(taint_info.get('func_name'))
    for func_info in functions:
        func_name = func_info.get('name')
        for taint_func_info in taint_funcs:
            if taint_func_info == func_name:
                taint_func_list.append(taint_func_info)

    # 保存操作日志
    LogRecords.save('',
                    category='query',
                    action='查询污点函数列表',
                    desc='查询指定固件文件(ID=%s)在代码分析中产生的函数列表' % file_id)

    return sys_app_ok_p({
        'taint_num': len(taint_func_list),
        'taint_func_list': taint_func_list
    })
Ejemplo n.º 7
0
def angr_convert_code(request):
    try:
        filename = req_get_param(request, 'filename')
        arch = getarch(filename)
        # load_options = {'auto_load_libs': False, 'main_opts': {'base_addr': 0}})
        # proj = angr.Project(filename, load_options={
        #     'main_opts': {
        #         'backend': 'blob',
        #         'base_addr': 0,
        #         'arch': arch,
        #     },
        # })

        # 装载二进制程序
        proj = angr.Project(filename,
                            load_options={
                                'auto_load_libs': False,
                                'main_opts': {
                                    'backend': 'blob',
                                    'base_addr': 0,
                                    'arch': arch,
                                },
                            })

        print(proj.arch)
        state = proj.factory.entry_state()

        print(proj.entry)
        #### Blocks # 转换入口点为基本块
        block = proj.factory.block(
            proj.entry)  # lift a block of code from the program's entry point
        pp = block.pp()  # pretty-print a disassembly to stdout
        print(block.instructions)  # how many instructions are there?
        print(block.instruction_addrs
              )  # what are the addresses of the instructions?
        print(block.capstone)  # capstone disassembly
        print(
            block.vex
        )  # VEX IRSB (that's a python internal address, not a program address)

        irsb = proj.factory.block(proj.entry).vex
        irsb.pp()
        irsb.next.pp()

    except binwalk.ModuleException as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({
        'code': str(block.vex),
    })
def test_zip_file(request):
    file_id = ReqParams.one(request, 'file_id')

    file_path = FwFilesStorage.export(file_id)
    if file_path is None:
        return sys_app_err('FW_FILE_NOT_FOUND')

    # compress_files = _test_zipfile(file_path)

    # compress_files = _test_patool(file_path)

    compress_files = _test_py7zr(file_path)

    return sys_app_ok_p(compress_files)
Ejemplo n.º 9
0
def poc_download(request):
    firmware_id = req_get_param(request, 'firmware_id')
    item = firmware_pocs.fetch(firmware_id)
    if item is None:
        return sys_app_err('ERROR_FWPOC_NOT_FOUND')

    file_name = item['aliases']
    # 对文本类型的文件名称增加txt后缀
    download_name = SysUtils.add_plain_text_file_suffix(file_name)
    # 设置响应内容的文件下载参数
    response = HttpResponse(item['content'], content_type='application/octet-stream')
    response['Content-Disposition'] = 'attachment;filename="%s"' % (urlquote(download_name))
    #SysLog.success('下载POC', '成功下载POC文件,漏洞ID={}'.format(firmware_id))
    return response
Ejemplo n.º 10
0
def binwalk_file_extract(request):
    filename = req_get_param(request, 'filename')
    try:
        list_temp = []
        # filename=US_W331AV1.0BR_V1.0.0.12_cn&en_TD.bin 文件名带特殊符号无法进行抽取文件
        for module in binwalk.scan(filename,
                                   signature=True,
                                   quiet=True,
                                   extract=True):
            for result in module.results:
                if result.file.path in module.extractor.output:
                    # These are files that binwalk carved out of the original firmware image, a la dd
                    if result.offset in module.extractor.output[
                            result.file.path].carved:
                        print
                        "Carved data from offset 0x%X to %s" % (
                            result.offset, module.extractor.output[
                                result.file.path].carved[result.offset])

                        list_temp.append(module.extractor.output[
                            result.file.path].carved[result.offset])
                    # These are files/directories created by extraction utilities (gunzip, tar, unsquashfs, etc)
                    if result.offset in module.extractor.output[
                            result.file.path].extracted:
                        if len(module.extractor.output[result.file.path].
                               extracted[result.offset].files):
                            print
                            "Extracted %d files from offset 0x%X to '%s' using '%s'" % (
                                len(module.extractor.output[result.file.path].
                                    extracted[result.offset].files),
                                result.offset,
                                module.extractor.output[result.file.path].
                                extracted[result.offset].files[0],
                                module.extractor.output[result.file.path].
                                extracted[result.offset].command)

                            list_temp.append(
                                module.extractor.output[result.file.path].
                                extracted[result.offset].files)

    except binwalk.ModuleException as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({'extract': 'ok', 'filelist': list_temp})
Ejemplo n.º 11
0
    def get_inverted_fw_data(self, index_con):
        index_con = InvertedIndex.str_to_hex(index_con)
        print(index_con)
        result = file_inverted_col.find({'index_con': {'$regex': index_con}})
        item_list = list(result)
        if item_list is None or len(item_list) == 0:
            return sys_app_ok()

        file_ids_str = ''
        for filePo in item_list:
            file_id = filePo.get('file_id')

            if file_ids_str.find(file_id) > -1:
                continue
            file_ids_str += file_id + ','

        file_ids = file_ids_str.split(',')

        results = []
        if len(file_ids) > 0:
            files_result = fw_files_col.find({
                'component': 1,
                'file_id': {
                    '$in': file_ids
                }
            })
            file_list = list(files_result)
            if file_list is None or len(file_list) == 0:
                return sys_app_err('ERROR_INVALID_PARAMETER')

            for file_info in file_list:
                file_info.pop('_id')
                pack_info = pack_files_col.find_one(
                    {'pack_id': file_info.get('pack_id')})
                if pack_info is not None:
                    pack_info.pop('_id')
                    file_info['pack_info'] = pack_info

            return sys_app_ok_p({'total': len(file_list), 'files': file_list})

        return sys_app_ok()
Ejemplo n.º 12
0
    def get_inverted_data(self, index_con, file_id):
        if index_con is not None and len(index_con) > 0:
            index_con = InvertedIndex.str_to_hex(index_con)
            result = file_inverted_col.find({
                'index_con': {
                    '$regex': index_con
                }
            }).limit(100)
            item_list = list(result)
        elif file_id is not None and len(file_id) > 0:
            result = file_inverted_col.find({'file_id': file_id}).limit(100)
            item_list = list(result)
        else:
            return sys_app_err('ERROR_INVALID_PARAMETER')

        if item_list is None or len(item_list) == 0:
            return sys_app_ok()

        for item_info in item_list:
            item_info.pop("_id")
        return sys_app_ok_p({'total': len(item_list), 'items': item_list})
Ejemplo n.º 13
0
def angr_convert2asm(request):
    insns = []
    asms = []
    try:
        filename = req_get_param(request, 'filename')
        arch = getarch(filename)
        p = angr.Project(filename,
                         load_options={
                             'auto_load_libs': False,
                             'main_opts': {
                                 'backend': 'blob',
                                 'base_addr': 0,
                                 'arch': arch,
                             },
                         })
        maxadd = p.loader.max_addr
        minadd = p.loader.min_addr
        print(minadd, maxadd)

        # let's disasm with capstone to search targets
        insn_bytes = p.loader.memory.load(0, maxadd)

        for cs_insn in p.arch.capstone.disasm(insn_bytes, 0):
            insns.append(CapstoneInsn(cs_insn))
            print("0x%x:\t%s\t\t%s" %
                  (cs_insn.address, cs_insn.mnemonic, cs_insn.op_str))
            # print(str(CapstoneInsn(cs_insn)))
        block = CapstoneBlock(0, insns, 0, p.arch)

        for ins in block.insns:
            asms.append(str(ins))
            # print(ins)

    except Exception as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({
        'ASM': asms,
    })
Ejemplo n.º 14
0
def poc_fetch(request):
    firmware_id = req_get_param(request, 'firmware_id')
    # doc = firmware_db.fetch(firmware_id)
    poc = firmware_pocs.fetch(firmware_id)
    if poc is None:
        return sys_app_err('ERROR_FWPOC_NOT_FOUND')
    print(poc['aliases'])
    # print(poc['firmware_path'])
    # print(poc['length'])
    # print(poc['filelist'])
    filepath = poc['firmware_path']
    filename = poc['filelist']
    length = poc['length']

    #SysLog.success('提取POC', '成功提取漏洞的POC(漏洞ID={})'.format(firmware_id))
    # doc['poc'] = poc

    # 将解压缩后的固件文件信息存入mongodb firmware_info
    item = {'fw_info': {'filepath': filepath, 'filename': filename, 'length': length}}
    firmware_db.update(firmware_id, item)

    return sys_app_ok_p(poc)
def cfg_func_list(request):
    # 从请求中取参数:文件 ID
    file_id = ReqParams.one(request, 'file_id')

    # 查找函数列表分析结果
    # 查询文件 CFG 分析的标记
    is_cfg = CfgAnalyzeService.has_cfg_analyze(file_id)
    if not is_cfg:
        # 启动分析任务
        task_id = CfgAnalyzeService.start_cfg_task(file_id)
        # 保存操作日志
        LogRecords.save({
            'task_id': task_id,
            'file_id': file_id
        },
                        category='analysis',
                        action='分析CFG',
                        desc='对二进制文件做调用流程图分析')

        # 返回响应:任务初始化的信息
        return sys_app_ok_p(MyTask.fetch_exec_info(task_id))

    # 启动分析任务
    functions = FilesService.functions_list(file_id)
    # 查找函数列表分析结果
    # functions = CfgAnalyzeResultDAO.get_functions(file_id)
    if len(functions) == 0:
        return sys_app_err(Error.FW_FILE_NO_CFG_ANALYZE)

    # 保存操作日志
    LogRecords.save('',
                    category='query',
                    action='查询函数列表',
                    desc='查询指定固件文件(ID=%s)在代码分析中产生的函数列表' % file_id)

    return sys_app_ok_p({
        'functions_count': len(functions),
        'functions': functions
    })
Ejemplo n.º 16
0
    def download_report(self, report_id):

        report_info = report_record_col.find_one({'report_id': report_id})
        if report_info is None:
            return sys_app_err('ERROR_INVALID_PARAMETER')

        store_path = report_info.get('pdf_path')
        pdf_name = report_info.get('pdf_name')

        def send_chunk():  # 流式读取
            with open(store_path, 'rb') as target_file:
                while True:
                    chunk = target_file.read(20 * 1024 * 1024)  # 每次读取20M
                    if not chunk:
                        break
                    yield chunk

        # 设置响应内容的文件下载参数
        response = HttpResponse(send_chunk(),
                                content_type='application/octet-stream')
        response['Content-Disposition'] = 'attachment;filename="%s"' % (
            urlquote(pdf_name))
        return response
Ejemplo n.º 17
0
    def save(self, pack_id):

        title_name = '固件分析报告'

        result_pack = pack_files_col.find({'pack_id': pack_id})

        pack_list = list(result_pack)

        firmware_name = ''
        firmware_file_num = 0
        execute_file_num = 0
        fw_file_lists = ''

        firmware_md5 = ''
        firmware_size = ''

        if pack_list is not None and len(pack_list) > 0:
            pack_info = pack_list[0]
            firmware_name = pack_info.get('name')
            pack_id = pack_info.get('pack_id')
            pack_file_id = pack_info.get('file_id')

            result_files = fw_files_col.find({'pack_id': pack_id})
            fw_file_lists = list(result_files)
            if fw_file_lists is not None or len(fw_file_lists) > 0:
                firmware_file_num = len(fw_file_lists)

                for file_info in fw_file_lists:
                    fw_file_type = file_info.get('file_type')

                    if fw_file_type == 4:
                        execute_file_num += 1

            item = PackFilesStorage.fetch(pack_file_id)

            firmware_md5 = item.get('md5')
            length_b = item.get('length')
            length_kb = length_b / 1024
            length_mb = length_kb / 1024
            if length_kb < 1:
                firmware_size = str('%.2f' % length_b) + ' B'
            elif length_mb < 1:
                firmware_size = str('%.2f' % length_kb) + ' KB'
            else:
                firmware_size = str('%.2f' % length_mb) + ' MB'
        else:
            return sys_app_err('ERROR_INVALID_PARAMETER')

        # firmware_inst = 'MIPS'
        # firmware_decomp_size = '7.2M'

        content = list()

        report_time = SysUtils.get_now_time_str()

        self.draw_con(content, title_name,
                      self.text_type(20, 30, colors.black, 1))
        self.draw_con(content, '报告生成时间:' + report_time,
                      self.text_type(11, 20, colors.black, 2))
        content.append(Spacer(300, 20))  # 添加空白,长度300,宽20

        # 1 固件分析综述
        self.summary_info(content, firmware_name, firmware_md5, firmware_size,
                          pack_id, firmware_file_num, execute_file_num)

        ct = self.text_type(10, 15, colors.black, 1)
        # 设置自动换行
        ct.wordWrap = 'CJK'

        # 2 组件关联的漏洞
        self.relation_loophole(content, fw_file_lists, firmware_name, ct)

        # 3 可执行文件详情
        self.fw_file_table(content, fw_file_lists, ct)

        # 4 特征码
        self.file_inverted_table(content, fw_file_lists, ct)

        self.draw_con(content, '报告结束', self.text_type(11, 20, colors.black, 1))

        time_stamp = SysUtils.parse_time_stamp_str()
        inde = firmware_name.index('.')

        if inde > -1:
            firmware_name = firmware_name[0:inde]

        pdf_name = firmware_name + title_name + time_stamp + '.pdf'

        path = './firmware_analyze_serv_report/'

        if not os.path.exists(path):
            os.mkdir(path)

        pdf_path = path + pdf_name
        # 生成pdf文件
        doc = SimpleDocTemplate(pdf_path, pagesize=letter)
        doc.build(content)

        report_id = StrUtils.uuid_str()
        report_record_info = {
            'report_id': report_id,
            'pack_id': pack_id,
            'pack_name': firmware_name,
            'pdf_path': pdf_path,
            'pdf_name': pdf_name,
            'create_time': report_time
        }

        report_record_col.save(report_record_info)

        return sys_app_ok()
Ejemplo n.º 18
0
def fwdownloadex(request):
    # print(Sys_code_err)
    print("run into fwdownload")

    # 获取下载URL
    downloadurl = req_get_param(request, 'url')
    print(downloadurl)
    savepath = settings.FW_PATH
    if os.path.isdir(savepath):
        pass
    else:
        os.mkdir(savepath)

    # 获取数据库固件ID
    firmware_id = firmware_db.get_suggest_firmware_id(None)
    item = {
        'firmware_id': firmware_id
            }

    # 任务ID 同固件ID
    task_id = firmware_db.get_suggest_task_id(None)
    task_item = {
        'task_id': task_id
            }

    task_item['task_id'] = task_id
    task_item['type'] = 'download'
    task_item['percentage'] = ''
    task_item['status'] = '0'
    firmware_db.task_add(task_item)

    # download_info = Mydownload.fwdownload(downloadurl,savepath)
    # print(download_info)

    try:
        """
        download file from internet
        :param url: path to download from
        :param savepath: path to save files
        :return: None
        """
        def reporthook(a, b, c):
            """
            显示下载进度
            :param a: 已经下载的数据块
            :param b: 数据块的大小
            :param c: 远程文件大小
            :return: None
            """
            percentage = round(a * b * 100.0 / c, 1)
            # print("\rdownloading: %5.1f%%" % (a * b * 100.0 / c), end="")
            print("\rdownloading: %5.1f%%" % percentage, end="")

            task_item['percentage'] = percentage
            firmware_db.task_update(task_id, task_item)

        filename = os.path.basename(downloadurl)
        # 判断是否为合法下载文件名 .zip .bin .img .rar .exe ...
        filetype = 'zip,bin,img,rar,exe'
        file_list = filename.split('.')
        result = file_list[file_list.__len__() - 1] in filetype
        print(result)
        if not result:
            #
            return sys_app_err_p('ERROR_FETCH_FILE_TYPE', {'filetype': file_list[file_list.__len__() - 1]})

        # 判断文件是否存在,如果不存在则下载
        if not os.path.isfile(os.path.join(savepath, filename)):
            print('Downloading data from %s' % downloadurl)
            # homepage = 'http://www.comfast.cn/uploadfile/%E8%BD%AF%E4%BB%B6%E9%A9%B1%E5%8A%A8/%E5%9B%BA%E4%BB%B6/OrangeOS-X86-V2.1.0_20170114.zip'
            # 'http://comfast.com.cn/upload/%E8%BD%AF%E4%BB%B6%E9%A9%B1%E5%8A%A8/%E5%9B%BA%E4%BB%B6/CF-AC101-V2.4.0.zip'
            'http://comfast.com.cn/upload/软件驱动/固件/CF-AC101-V2.4.0.zip'
            'http://www.comfast.cn/uploadfile/firmware/CF-AC101-V2.6.1.zip'
            # homepage = homepage.encode()
            print(downloadurl)

            urlretrieve(downloadurl, os.path.join(savepath, filename), reporthook=reporthook)

            item['fw_file_name'] = filename
            item['application_mode'] = file_list[0]
            item['fw_manufacturer'] = ''
            firmware_db.add(item)

            # task_item['type'] = file_list[0]
            # task_item['status'] = ''
            # task_item['remark'] = ''
            # firmware_db.task_add(task_item)

            pathfilename = savepath+"\\"+filename
            with open(pathfilename, 'rb') as myimage:
                data = myimage.read()
                firmware_pocs.add(firmware_id, filename, data)

            print('\nDownload finished!')
        else:
            print('File already exsits!')
        # 获取文件大小
        filesize = os.path.getsize(os.path.join(savepath, filename))
        # 文件大小默认以Bytes计, 转换为Mb
        print('File size = %.2f Mb' % (filesize / 1024 / 1024))
        return sys_app_ok_p('ERROR_OK')
    except Exception as e:
        print(e)
        return sys_app_err(e)
Ejemplo n.º 19
0
    def inverted(self, file_id):

        file_result = fw_files_col.find({'file_id': file_id})
        file_list = list(file_result)

        if file_list is None or len(file_list) == 0:
            return sys_app_err('ERROR_INVALID_PARAMETER')

        filePo = file_list[0]
        file_path = filePo.get('file_path')

        # file_path = 'E:/samples/argv_test'
        # file_path = 'E:/samples/py_code.txt'

        dict1 = {}
        dict2 = {}
        sentences = InvertedIndex.read_file(self, file_path)

        sentencesLen = len(sentences)

        for i in range(sentencesLen):
            sentence = sentences[i]
            for word in sentence:
                if word == '':
                    continue
                if word.lower() not in dict1:
                    dict1[word.lower()] = set()  # new word
                    dict2[word.lower()] = 1
                else:
                    dict2[word.lower()] += 1
                dict1[word.lower()].add(i + 1)  # update for dictionary

        answer_list = sorted(dict2.items(), key=lambda d: d[1],
                             reverse=True)  # Sort by wordcount of dictionary.
        answer_sort_ascll = sorted(answer_list, key=lambda x: x[0])

        for word in answer_sort_ascll:
            word0 = InvertedIndex.str_to_hex(word[0]).replace('/x0', '')
            sort_dotid = sorted(dict1[word[0]])

            position = ''
            for i in range(len(sort_dotid)):
                position += str(sort_dotid[i])
                if i != (len(sort_dotid) - 1):
                    position += ','

            index_con = word0
            index_con_str = InvertedIndex.hex_to_str(word0)
            appear_total = word[1]

            vulner_info = {
                'file_id': file_id,
                'file_path': file_path,
                'index_con': index_con,
                'appear_total': appear_total,
                'position': position
            }

            result = file_inverted_col.find({
                'file_id': file_id,
                'index_con': index_con,
                'appear_total': appear_total
            })
            item_list = list(result)

            if (item_list is None or len(item_list)
                    == 0) and len(index_con) > 0 and len(index_con_str) > 10:
                file_inverted_col.save(vulner_info)

        # 对组件列表增加建立 inverted 完成标志
        FwFileDO.set_inverted(file_id)

        return sys_app_ok()
Ejemplo n.º 20
0
def binwalk_file_extractEx(request):
    # filename = req_get_param(request, 'filename')
    firmware_id = req_get_param(request, 'firmware_id')
    try:
        # 查询数据库 得到固件名
        fw = firmware_db.fetch(firmware_id)

        # todo check fw is NULL
        if fw['fw_info']['filepath'] is not None:
            filename = fw['fw_info']['filepath'] + fw['fw_info']['filename'][0]
        else:
            return sys_app_ok_p({'decode': 'Null', 'description': "解析文件名出错"})

        list_temp = []

        # filename=US_W331AV1.0BR_V1.0.0.12_cn&en_TD.bin 文件名带特殊符号无法进行抽取文件
        for module in binwalk.scan(filename,
                                   signature=True,
                                   quiet=True,
                                   extract=True):
            for result in module.results:
                if result.file.path in module.extractor.output:
                    # These are files that binwalk carved out of the original firmware image, a la dd
                    if result.offset in module.extractor.output[
                            result.file.path].carved:
                        print
                        "Carved data from offset 0x%X to %s" % (
                            result.offset, module.extractor.output[
                                result.file.path].carved[result.offset])

                        list_temp.append(module.extractor.output[
                            result.file.path].carved[result.offset])
                    # These are files/directories created by extraction utilities (gunzip, tar, unsquashfs, etc)
                    if result.offset in module.extractor.output[
                            result.file.path].extracted:
                        if len(module.extractor.output[result.file.path].
                               extracted[result.offset].files):
                            print
                            "Extracted %d files from offset 0x%X to '%s' using '%s'" % (
                                len(module.extractor.output[result.file.path].
                                    extracted[result.offset].files),
                                result.offset,
                                module.extractor.output[result.file.path].
                                extracted[result.offset].files[0],
                                module.extractor.output[result.file.path].
                                extracted[result.offset].command)

                            list_temp.append(
                                module.extractor.output[result.file.path].
                                extracted[result.offset].files)

        # 将抽取的文件信息存入mongodb firmware_info
        dic = {}
        item = {}
        index = 0
        for off_set in list_temp:
            index += 1
            filex = 'file' + str(index)
            dic[filex] = list_temp[index - 1]

        item['extract_info'] = dic
        firmware_db.update(firmware_id, item)

    except binwalk.ModuleException as e:
        print("Critical failure:", e)
        return sys_app_err('ERROR_INTERNAL_ERROR')
    return sys_app_ok_p({'extract': 'ok', 'filelist': list_temp})