def main(): # global step, best_prec1, model, crit, optimizer, saver, writer # global train_dataset, test_dataset, train_loader, test_loader args = parse_args() if args.gpu_id: torch.cuda.set_device(args.gpu_id[0]) context = dict(args=args) print('prepare dataset...') (train_dataset, train_loader), (test_dataset, test_loader) = prepare_dataset(context) print('prepare model...') model, crit, optimizer = prepare_model(context) print('prepare logger...') writer, saver = prepare_logger(context) step, best_prec1, stats_train, stats_test, timer = prepare_stats(context) load_checkpoint(context) print('start training...') train(context)
def main(args=sys.argv[1:]): args = parse_args() commandline.setup_logging('raptor', args, {'tbpl': sys.stdout}) LOG = get_default_logger(component='raptor-main') LOG.info("raptor-start") LOG.info("received command line arguments: %s" % str(args)) # if a test name specified on command line, and it exists, just run that one # otherwise run all available raptor tests that are found for this browser raptor_test_list = get_raptor_test_list(args, mozinfo.os) # ensure we have at least one valid test to run if len(raptor_test_list) == 0: LOG.critical("abort: no tests found") sys.exit(1) LOG.info("raptor tests scheduled to run:") for next_test in raptor_test_list: LOG.info(next_test['name']) raptor = Raptor(args.app, args.binary, args.run_local, args.obj_path) raptor.start_control_server() for next_test in raptor_test_list: if 'page_timeout' not in next_test.keys(): next_test['page_timeout'] = 120000 if 'page_cycles' not in next_test.keys(): next_test['page_cycles'] = 1 raptor.run_test(next_test, timeout=int(next_test['page_timeout'])) success = raptor.process_results() raptor.clean_up() if not success: # didn't get test results; test timed out or crashed, etc. we want job to fail LOG.critical("error: no raptor test results were found") os.sys.exit(1)
print 'Task Error.' url_queue.task_done() # print url_queue.empty() def main(num_t): threads = [] for x in xrange(num_t): threads.append(ScanThread()) threads[x].start() for y in threads: y.join() if __name__ == '__main__': server = 'http://127.0.0.1:8775' headers = {'Content-Type': 'application/json'} sql_list = [] arg = parse_args() url_path = arg.u num_thread = arg.t url_queue = Queue() scan_queue(url_path) main(num_thread) print "\n------------------SQL Injections-------------------" print sql_list print "\n--------------------It's done.---------------------"
# 所以用 get_text() print each.get_text() def main(s_num, e_num, t_num): threads = [] for i in range(s_num, e_num): get_url(i) print url_queue.qsize() for x in range(t_num): threads.append(IVSThread()) # threads[x].setDaemon(True) threads[x].start() for z in range(t_num): threads[z].join() # url_queue.join() 如果用队列来阻塞主线程的话,需要在非忽略的get也task_done if __name__ == "__main__": start_time = time.time() # 创建命令行参数 Namespace对象 args = parse_args() main(args.s, args.e, args.t) # print "--------------------It's done.---------------------" # cost_time = time.time() - start_time # print "Current spider is finished in %d mins %.2f seconds." % (cost_time / 60, cost_time % 60)
def main(args=sys.argv[1:]): args = parse_args() args.extra_prefs = parse_preferences(args.extra_prefs or []) if args.enable_fission: args.extra_prefs.update({ "fission.autostart": True, "dom.serviceWorkers.parent_intercept": True, "browser.tabs.documentchannel": True, }) if args.extra_prefs and args.extra_prefs.get("fission.autostart", False): args.enable_fission = True commandline.setup_logging("raptor", args, {"tbpl": sys.stdout}) LOG.info("Python version: %s" % sys.version) LOG.info("raptor-start") if args.debug_mode: LOG.info("debug-mode enabled") LOG.info("received command line arguments: %s" % str(args)) # if a test name specified on command line, and it exists, just run that one # otherwise run all available raptor tests that are found for this browser raptor_test_list = get_raptor_test_list(args, mozinfo.os) raptor_test_names = [ raptor_test["name"] for raptor_test in raptor_test_list ] # ensure we have at least one valid test to run if len(raptor_test_list) == 0: LOG.critical("test '{}' could not be found for {}".format( args.test, args.app)) sys.exit(1) LOG.info("raptor tests scheduled to run:") for next_test in raptor_test_list: LOG.info(next_test["name"]) if not args.browsertime: if args.app == "firefox": raptor_class = WebExtensionFirefox elif args.app in CHROMIUM_DISTROS: raptor_class = WebExtensionDesktopChrome else: raptor_class = WebExtensionAndroid else: def raptor_class(*inner_args, **inner_kwargs): outer_kwargs = vars(args) # peel off arguments that are specific to browsertime for key in outer_kwargs.keys(): if key.startswith("browsertime_"): value = outer_kwargs.pop(key) inner_kwargs[key] = value if args.app == "firefox" or args.app in CHROMIUM_DISTROS: klass = BrowsertimeDesktop else: klass = BrowsertimeAndroid return klass(*inner_args, **inner_kwargs) try: raptor = raptor_class( args.app, args.binary, run_local=args.run_local, noinstall=args.noinstall, installerpath=args.installerpath, obj_path=args.obj_path, gecko_profile=args.gecko_profile, gecko_profile_interval=args.gecko_profile_interval, gecko_profile_entries=args.gecko_profile_entries, symbols_path=args.symbols_path, host=args.host, power_test=args.power_test, cpu_test=args.cpu_test, memory_test=args.memory_test, live_sites=args.live_sites, cold=args.cold, is_release_build=args.is_release_build, debug_mode=args.debug_mode, post_startup_delay=args.post_startup_delay, activity=args.activity, intent=args.intent, interrupt_handler=SignalHandler(), enable_webrender=args.enable_webrender, extra_prefs=args.extra_prefs or {}, device_name=args.device_name, no_conditioned_profile=args.no_conditioned_profile, disable_perf_tuning=args.disable_perf_tuning, conditioned_profile_scenario=args.conditioned_profile_scenario, chimera=args.chimera, project=args.project, verbose=args.verbose) except Exception: traceback.print_exc() LOG.critical( "TEST-UNEXPECTED-FAIL: could not initialize the raptor test runner" ) os.sys.exit(1) success = raptor.run_tests(raptor_test_list, raptor_test_names) if not success: # if we have results but one test page timed out (i.e. one tp6 test page didn't load # but others did) we still dumped PERFHERDER_DATA for the successfull pages but we # want the overall test job to marked as a failure pages_that_timed_out = raptor.get_page_timeout_list() if pages_that_timed_out: for _page in pages_that_timed_out: message = [ ("TEST-UNEXPECTED-FAIL", "test '%s'" % _page["test_name"]), ("timed out loading test page", "waiting for pending metrics"), ] if _page.get("pending_metrics") is not None: LOG.warning("page cycle {} has pending metrics: {}".format( _page["page_cycle"], _page["pending_metrics"])) LOG.critical(" ".join("%s: %s" % (subject, msg) for subject, msg in message)) else: # we want the job to fail when we didn't get any test results # (due to test timeout/crash/etc.) LOG.critical( "TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" % ", ".join(raptor_test_names)) os.sys.exit(1) # if we're running browsertime in the CI, we want to zip the result dir if args.browsertime and not args.run_local: result_dir = raptor.results_handler.result_dir() if os.path.exists(result_dir): LOG.info("Creating tarball at %s" % result_dir + ".tgz") with tarfile.open(result_dir + ".tgz", "w:gz") as tar: tar.add(result_dir, arcname=os.path.basename(result_dir)) LOG.info("Removing %s" % result_dir) shutil.rmtree(result_dir) # when running raptor locally with gecko profiling on, use the view-gecko-profile # tool to automatically load the latest gecko profile in profiler.firefox.com if args.gecko_profile and args.run_local: if os.environ.get("DISABLE_PROFILE_LAUNCH", "0") == "1": LOG.info( "Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1" ) else: view_gecko_profile_from_raptor()
def main(args=sys.argv[1:]): args = parse_args() commandline.setup_logging('raptor', args, {'tbpl': sys.stdout}) LOG = get_default_logger(component='raptor-main') LOG.info("raptor-start") if args.debug_mode: LOG.info("debug-mode enabled") LOG.info("received command line arguments: %s" % str(args)) # if a test name specified on command line, and it exists, just run that one # otherwise run all available raptor tests that are found for this browser raptor_test_list = get_raptor_test_list(args, mozinfo.os) raptor_test_names = [raptor_test['name'] for raptor_test in raptor_test_list] # ensure we have at least one valid test to run if len(raptor_test_list) == 0: LOG.critical("abort: no tests found") sys.exit(1) LOG.info("raptor tests scheduled to run:") for next_test in raptor_test_list: LOG.info(next_test['name']) if args.app == "firefox": raptor_class = RaptorDesktopFirefox elif args.app == "chrome": raptor_class = RaptorDesktopChrome else: raptor_class = RaptorAndroid raptor = raptor_class(args.app, args.binary, run_local=args.run_local, obj_path=args.obj_path, gecko_profile=args.gecko_profile, gecko_profile_interval=args.gecko_profile_interval, gecko_profile_entries=args.gecko_profile_entries, symbols_path=args.symbols_path, host=args.host, power_test=args.power_test, is_release_build=args.is_release_build, debug_mode=args.debug_mode, activity=args.activity) raptor.create_browser_profile() raptor.create_browser_handler() raptor.start_control_server() for next_test in raptor_test_list: raptor.run_test(next_test, timeout=int(next_test['page_timeout'])) success = raptor.process_results(raptor_test_names) raptor.clean_up() if not success: # didn't get test results; test timed out or crashed, etc. we want job to fail LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" % ', '.join(raptor_test_names)) os.sys.exit(1) # if we have results but one test page timed out (i.e. one tp6 test page didn't load # but others did) we still dumped PERFHERDER_DATA for the successfull pages but we # want the overall test job to marked as a failure pages_that_timed_out = raptor.get_page_timeout_list() if len(pages_that_timed_out) > 0: for _page in pages_that_timed_out: LOG.critical("TEST-UNEXPECTED-FAIL: test '%s' timed out loading test page: %s" % (_page['test_name'], _page['url'])) os.sys.exit(1) # when running raptor locally with gecko profiling on, use the view-gecko-profile # tool to automatically load the latest gecko profile in profiler.firefox.com if args.gecko_profile and args.run_local: if os.environ.get('DISABLE_PROFILE_LAUNCH', '0') == '1': LOG.info("Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1") else: view_gecko_profile(args.binary)
if url_queue.empty(): break scan_url = url_queue.get() scan_resutl = wvs_scan(scan_url) (code, save_load) = scan_resutl.split('|') if code > 0: xml_result = parse_xml(save_load + '\\export.xml') # str.join(sequence),序列之间用str间隔,这里用换行来间隔转换成字符串 send_main(mail_list, 'WvsScanner Report--'+scan_url, '\n'.join(xml_result)) url_queue.task_done() def main(url_l, t_num): read_url(url_l) thread = [] for x in range(t_num): thread.append(ScanThread()) thread[x].start() for i in thread: if i.isAlive(): i.join() if __name__ == "__main__": arg = parse_args() url_list = arg.u cmd_num = arg.t main(url_list, cmd_num)
url_ignored = "http://www.wooyun.org/" + url return url_res(url_ignored).find("忽略") def vul_detail(url): url_vul = "http://www.wooyun.org/" + url soup = BeautifulSoup(url_res(url_vul), 'html.parser', from_encoding='UTF-8') # print soup.find_all('title')[0].string for each in soup.find_all('code'): # 因为each.string 无法获取其内包含多个子节点的内容,返回None # 所以用 get_text() print each.get_text() def main(s_num, e_num): for i in range(s_num, e_num): get_url(i) print "-------------------It's done.--------------------" if __name__ == "__main__": start_time = time.time() # 创建命令行参数 Namespace对象 args = parse_args() main(args.s, args.e) cost_time = time.time() - start_time print "Current spider is finished in " + str(int(cost_time / 60)) + " mins %.2f seconds." % (cost_time % 60)
def main(args=sys.argv[1:]): args = parse_args() commandline.setup_logging('raptor', args, {'tbpl': sys.stdout}) LOG = get_default_logger(component='raptor-main') LOG.info("raptor-start") LOG.info("received command line arguments: %s" % str(args)) # if a test name specified on command line, and it exists, just run that one # otherwise run all available raptor tests that are found for this browser raptor_test_list = get_raptor_test_list(args, mozinfo.os) # ensure we have at least one valid test to run if len(raptor_test_list) == 0: LOG.critical("abort: no tests found") sys.exit(1) LOG.info("raptor tests scheduled to run:") for next_test in raptor_test_list: LOG.info(next_test['name']) raptor = Raptor(args.app, args.binary, args.run_local, args.obj_path, args.gecko_profile, args.gecko_profile_interval, args.gecko_profile_entries, args.symbols_path) raptor.start_control_server() for next_test in raptor_test_list: if 'page_timeout' not in next_test.keys(): next_test['page_timeout'] = 120000 if 'page_cycles' not in next_test.keys(): next_test['page_cycles'] = 1 raptor.run_test(next_test, timeout=int(next_test['page_timeout'])) success = raptor.process_results() raptor.clean_up() if not success: # didn't get test results; test timed out or crashed, etc. we want job to fail LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found") os.sys.exit(1) # if we have results but one test page timed out (i.e. one tp6 test page didn't load # but others did) we still dumped PERFHERDER_DATA for the successfull pages but we # want the overall test job to marked as a failure pages_that_timed_out = raptor.get_page_timeout_list() if len(pages_that_timed_out) > 0: for _page in pages_that_timed_out: LOG.critical("TEST-UNEXPECTED-FAIL: test '%s' timed out loading test page: %s" % (_page['test_name'], _page['url'])) os.sys.exit(1) # when running raptor locally with gecko profiling on, use the view-gecko-profile # tool to automatically load the latest gecko profile in perf-html.io if args.gecko_profile and args.run_local: if os.environ.get('DISABLE_PROFILE_LAUNCH', '0') == '1': LOG.info("Not launching perf-html.io because DISABLE_PROFILE_LAUNCH=1") else: view_gecko_profile(args.binary)