def createReport(self, index, z, point): """Create report file. Prints error when report txt file cannot be opened for writing. """ try: try: # python 3.x report = open(self.reportFileName, 'w', encoding='utf-8') except: # python 2.x report = codecs.open(self.reportFileName, 'w', encoding='utf-8') except IOError as e: self.computeMessage.emit( u'Error', u'Unable open {} for writing. Reason: {}'.format( self.reportFileName, e), 'CRITICAL') return if index == 0: report_text = (u'/{}BQM2'.format(z), u'/MGRS:{}'.format(point, ls=os.linesep)) elif index == 1: report_text = (u'/{}CGH'.format(z), u'/MGRS:{}'.format(point, ls=os.linesep)) for line in report_text: report.write(line) report.close()
def decompile(path): """ Converts DEX to JAR(containing class files) and then class files to near original java code using 3 different decompilers and selecting the best available decompiled code """ common.pathToDEX = path pathToDex2jar = common.rootDir + "/lib/dex2jar/dex2jar.sh" sp = subprocess.Popen([pathToDex2jar, common.pathToDEX], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = sp.communicate() common.pathToJar = common.pathToDEX.rsplit(".",1)[0] + "_dex2jar.jar" dirname, extension = common.pathToJar.rsplit(".",1) zf = zipfile.ZipFile(common.pathToJar) #Total number of class files that need to be decompiled total_files = len(zf.namelist()) report.write("totalfiles", total_files) common.count = len([s for s in zf.namelist() if ((".class" in s) and ("$" not in s))]) pub.subscribe(decompiler_update, 'decompile') thread0 = Process(name='clear', target=clear, args = ()) thread1 = Process(name='jdcore', target=jdcore, args = (zf.filename,dirname)) thread2 = Process(name='procyon', target=cfr, args = (zf.filename,dirname)) thread3 = Process(name='cfr', target=procyon, args = (zf.filename,dirname)) thread0.start() thread0.join() progressbar1.start() progressbar2.start() progressbar3.start() thread1.start() thread2.start() thread3.start() thread1.join(0) thread2.join(0) thread3.join(0) with common.term.cbreak(): val = None while val not in (u'c', u'C'): with common.term.location(0,common.term.height-3): print "Decompilation may hang/take too long (usually happens when the source is obfuscated)." print "At any time," + common.term.bold_underline_red_on_white('Press C to continue') + " and QARK will attempt to run SCA on whatever was decompiled." val = common.term.inkey(timeout=1) if not (thread1.is_alive() or thread2.is_alive() or thread3.is_alive()): break if thread1.is_alive(): thread1.terminate() if thread2.is_alive(): thread2.terminate() if thread3.is_alive(): thread3.terminate() #Go back to the bottom of the screen with common.term.location(0,common.term.height): print "" g1 = grep_1(dirname, "// Byte code:") g2 = grep_1(dirname+"1", "// This method has failed to decompile.") g3 = grep_1(dirname+"2", "// This method could not be decompiled.") #print list(set(g1) - set(g2)) logger.info("Trying to improve accuracy of the decompiled files") restored = 0 try: for filename in g1: relative_filename = str(filename).split(dirname)[1] if any(relative_filename in s for s in g2): if any(relative_filename in s for s in g3): logger.debug("Failed to reconstruct: " + relative_filename) else: shutil.copy(dirname+"2"+relative_filename, filename) restored = restored +1 else: shutil.copy(dirname+"1"+relative_filename, filename) restored = restored +1 except Exception as e: print e.message report.write("restorestats","Restored " + str(restored) + " file(s) out of " + str(len(g1)) + " corrupt file(s)") logger.info("Restored " + str(restored) + " file(s) out of " + str(len(g1)) + " corrupt file(s)") logger.debug("Deleting redundant decompiled files") try: shutil.rmtree(dirname+"1") logger.debug("Deleted " + dirname+"1") shutil.rmtree(dirname+"2") logger.debug("Deleted " + dirname+"2") except Exception as e: logger.debug("Unable to delete redundant decompiled files (no impact on scan results): " + str(e))
for N in Nlist: W = dense_graph_random(N, dtype) searcher = pkg.dense_graph_bf_searcher(W, dtype=dtype) result = benchmark.search(searcher) results.append((N, ) + result) return results def run_bipartite_graph_benchmark(Nlist, dtype, pkg): results = [] for N in Nlist: b0, b1, W = bipartite_graph_random(N / 2, N / 2, dtype) searcher = pkg.bipartite_graph_bf_searcher(b0, b1, W, dtype=dtype) result = benchmark.search(searcher) results.append((N, ) + result) return results if __name__ == '__main__': # Nlist = [ 8, 16, 20, 24, 28, 32, 36 ] benchmark.duration = 10. Nlist = [8, 16, 20] results = run_dense_graph_benchmark(Nlist, np.float32, sq.cpu) report.write('dense_graph_bf_searcher.csv', results) results = run_bipartite_graph_benchmark(Nlist, np.float32, sq.cpu) report.write('bipartite_graph_bf_searcher.csv', results)
elif k == "trace": trace.enabled = eval(v) elif k == "dir": output = path(v) elif k == "file": res_fname = path(v) elif k == "series": series_fname = path(v) else: args[k] = eval(v) except: pass # none = arg m = model(**args) m.run() r = m.calc_results() #print results report.printr(r) report.write(r, output/res_fname) report.write_series(m, output/series_fname) #from guppy import hpy #h = hpy() #print h.heap()
def decompile(path): """ Converts DEX to JAR(containing class files) and then class files to near original java code using 3 different decompilers and selecting the best available decompiled code """ common.pathToDEX = path pathToDex2jar = common.rootDir + "/lib/dex2jar/dex2jar.sh" sp = subprocess.Popen([pathToDex2jar, common.pathToDEX], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = sp.communicate() common.pathToJar = common.pathToDEX.rsplit(".", 1)[0] + "_dex2jar.jar" dirname, extension = common.pathToJar.rsplit(".", 1) zf = zipfile.ZipFile(common.pathToJar) #Total number of class files that need to be decompiled total_files = len(zf.namelist()) report.write("totalfiles", total_files) common.count = len( [s for s in zf.namelist() if ((".class" in s) and ("$" not in s))]) pub.subscribe(decompiler_update, 'decompile') thread0 = Process(name='clear', target=clear, args=()) thread1 = Process(name='jdcore', target=jdcore, args=(zf.filename, dirname)) thread2 = Process(name='procyon', target=cfr, args=(zf.filename, dirname)) thread3 = Process(name='cfr', target=procyon, args=(zf.filename, dirname)) thread0.start() thread0.join() progressbar1.start() progressbar2.start() progressbar3.start() thread1.start() thread2.start() thread3.start() thread1.join(0) thread2.join(0) thread3.join(0) with common.term.cbreak(): val = None while val not in (u'c', u'C'): with common.term.location(0, common.term.height - 3): print "Decompilation may hang/take too long (usually happens when the source is obfuscated)." print "At any time," + common.term.bold_underline_red_on_white( 'Press C to continue' ) + " and QARK will attempt to run SCA on whatever was decompiled." val = common.term.inkey(timeout=1) if not (thread1.is_alive() or thread2.is_alive() or thread3.is_alive()): break if thread1.is_alive(): thread1.terminate() if thread2.is_alive(): thread2.terminate() if thread3.is_alive(): thread3.terminate() #Go back to the bottom of the screen with common.term.location(0, common.term.height): print "" g1 = grep_1(dirname, "// Byte code:") g2 = grep_1(dirname + "1", "// This method has failed to decompile.") g3 = grep_1(dirname + "2", "// This method could not be decompiled.") #print list(set(g1) - set(g2)) logger.info("Trying to improve accuracy of the decompiled files") restored = 0 try: for filename in g1: relative_filename = str(filename).split(dirname)[1] if any(relative_filename in s for s in g2): if any(relative_filename in s for s in g3): logger.debug("Failed to reconstruct: " + relative_filename) else: shutil.copy(dirname + "2" + relative_filename, filename) restored = restored + 1 else: shutil.copy(dirname + "1" + relative_filename, filename) restored = restored + 1 except Exception as e: print e.message report.write( "restorestats", "Restored " + str(restored) + " file(s) out of " + str(len(g1)) + " corrupt file(s)") logger.info("Restored " + str(restored) + " file(s) out of " + str(len(g1)) + " corrupt file(s)") logger.debug("Deleting redundant decompiled files") try: shutil.rmtree(dirname + "1") logger.debug("Deleted " + dirname + "1") shutil.rmtree(dirname + "2") logger.debug("Deleted " + dirname + "2") except Exception as e: logger.debug( "Unable to delete redundant decompiled files (no impact on scan results): " + str(e))
chars_counter += len(news_text) orig_path = get_path(news_date, title, 'txt', version='original_text') text_lemma_path = get_path(news_date, title, 'txt', version='lemm_text') xml_path = get_path(news_date, title, 'xml', version='xml') html_path = get_path(news_date, title, 'html', version='html') # сохраняем текст статьи, но пока без шапки, чтобы mystem не работал с # лишними данными save(news_text, orig_path) # необходимо для рекурсивного создания дирректорий, в ином случае данные # лемматизации некуда будет сохранять save('', text_lemma_path) save('', xml_path) lemmatisation(orig_path, xml_path, 'xml') lemmatisation(orig_path, text_lemma_path, 'text') # пересохраняем текст статьи уже с шапкой, т.к. лемматизация окончена text = get_original_text(news_text, title, news_date, news_link, author) save(text, orig_path) # сохранчем html страницу save(raw_news_page.decode('utf8'), html_path) # добавляем данные о новости в отчёт report.write(orig_path, title, news_date, news_link, author)
def main(args): # Load binary and source flow graphs exe = fparser.Executable(args.bin_json, args.dwarf_json, args.src_csv, args.optime_csv, simplify=(not args.no_simplify)) log.debug("Optime_csv={}".format(args.optime_csv)) ########################## # Load external loop info ########################## annot_file = None if args.annot_file is not None: with open(args.annot_file, 'r') as fp: annot_file = json.load(fp) log.info("External annot file loaded.") log.info(json.dumps(annot_file, indent=4)) ######################### # Process flows pairwise ######################### time_all_begin = time.time() funcs_all = set() funcs_mapped = dict() n_tot = n_grp = n_prec = 0 for bFlow, sFlow in exe.get_flow_pairs(): funcs_all.add(bFlow.name) log.debug("Mapping {} ({}) to {} ({}).".format(bFlow.name, bFlow.file, sFlow.name, sFlow.file)) if args.render_graphs: do_render_flows(bFlow=bFlow, sFlow=sFlow) annot_func = None if annot_file is not None and bFlow.name in annot_file: annot_func = annot_file[bFlow.name] ########## # mapping ########## try: full_map, rpt = do_mapping(bFlow=bFlow, sFlow=sFlow, annot_func=annot_func, hom_order=args.hom_order, mapper_name=args.mapper, do_render=args.render_graphs, trust_dbg=args.trust_dbg_info) funcs_mapped[bFlow.name] = full_map precise_map = _get_last_precise_map(full_map) stats = precise_map.calc_statistics() if stats is not None: n_grp += stats.data.get('graphs', 0) n_tot += stats.data.get('total', 0) n_prec += stats.data.get('mapped', 0) except AssertionError: full_map = rpt = None log.error("Failed to match flow {}.".format(bFlow.name), exc_info=True) # exit(2) ########## # outputs ########## # writes the mapping CSV and JSON reports if full_map: report.write(bFlow=bFlow, sFlow=sFlow, reportdata=rpt, mapping=full_map) if args.render_graphs: full_map.name = bFlow.name try: do_render_mapping(bFlow=bFlow, sFlow=sFlow, annot_file=annot_file, hierarchical_map=full_map) except AssertionError: log.warning("Error during rendering of mapping {}".format( full_map.name)) import traceback traceback.print_exc() ##################### # Overall Statistics ##################### funcs_unmapped = funcs_all.difference(set(funcs_mapped.keys())) time_all_end = time.time() percent_precise = (100. * n_prec) / n_tot if n_tot > 0 else 100. log.info("Statistics: {} subgraphs with {} nodes, {:.2f}% precise".format( n_grp, n_tot, percent_precise)) log.info("Mapped {} out of {} functions in {:.2f} seconds".format( len(funcs_mapped), len(funcs_all), time_all_end - time_all_begin)) if len(funcs_unmapped) > 0: log.warning("Unmapped functions: {}".format(", ".join( sorted(list(funcs_unmapped))))) # -- return len(funcs_unmapped)