def cleanup_visit(self): """Kill sniffer and Tor browser if they're running.""" wl_log.info("Cleaning up visit.") wl_log.info("Cancelling timeout") ut.cancel_timeout() if self.sniffer and self.sniffer.is_recording: wl_log.info("Stopping sniffer...") self.sniffer.stop_capture() # remove non-tor traffic self.filter_guards_from_pcap() if self.tb_driver and self.tb_driver.is_running: # shutil.rmtree(self.tb_driver.prof_dir_path) wl_log.info("Quitting selenium driver...") self.tb_driver.quit() # close all open streams to prevent pollution self.tor_controller.close_all_streams() if self.xvfb and not cm.running_in_CI: wl_log.info("Stopping display...") self.vdisplay.stop() # after closing driver and stoping sniffer, we run postcrawl self.post_crawl()
def test_cancel_timeout(self): ut.timeout(1) ut.cancel_timeout() try: sleep(1.1) except ut.TimeExceededError: self.fail("Cannot cancel timeout")
def test_cancel_timeout(self): start_time = time() ut.timeout(1) sleep(0.3) ut.cancel_timeout() elapsed_time = time() - start_time self.assertLess(elapsed_time, 5, 'Cancel time does not work. %s' % elapsed_time)
def close_all_streams(self): """Close all streams of a controller.""" print("Closing all streams") try: ut.timeout(cm.STREAM_CLOSE_TIMEOUT) for stream in self.controller.get_streams(): print("Closing stream %s %s %s " % (stream.id, stream.purpose, stream.target_address)) self.controller.close_stream(stream.id) # MISC reason except ut.TimeExceededError: print("Closing streams timed out!") except: print("Exception closing stream") finally: ut.cancel_timeout()
def close_all_streams(self): """Close all streams of a controller.""" wl_log.debug("Closing all streams") try: ut.timeout(cm.STREAM_CLOSE_TIMEOUT) for stream in self.controller.get_streams(): wl_log.debug( "Closing stream %s %s %s " % (stream.id, stream.purpose, stream.target_address)) self.controller.close_stream(stream.id) # MISC reason except ut.TimeExceededError: wl_log.critical("Closing streams timed out!") except: wl_log.debug("Exception closing stream") finally: ut.cancel_timeout()
def cleanup_visit(self): """Kill sniffer and Tor browser if they're running.""" wl_log.info("Cleaning up visit.") wl_log.info("Cancelling timeout") ut.cancel_timeout() if self.sniffer and self.sniffer.is_recording: wl_log.info("Stopping sniffer...") self.sniffer.stop_capture() if self.tb_driver and self.tb_driver.is_running: # shutil.rmtree(self.tb_driver.prof_dir_path) wl_log.info("Quitting selenium driver...") self.tb_driver.quit() # close all open streams to prevent pollution self.tor_controller.close_all_streams() if self.xvfb and not cm.running_in_CI: self.vdisplay.stop()
def launch_tor_with_custom_stem(datalist, browser): print("length of data: ", len(datalist)) tor_binary = join(cm.TorProxypath, cm.DEFAULT_TOR_BINARY_PATH) tor_process, controller = 0, 0 try: TRYTOR_CNT = cm.TRYCNT while TRYTOR_CNT > 0 and tor_process == 0 and controller == 0: print("try to setup tor:", str(TRYTOR_CNT)) tor_process, controller = TorSetup(tor_binary) TRYTOR_CNT -= 1 if tor_process == 0: raise TorSetupError print("finish tor proxy setup...") xvfb_display = start_xvfb() # virtual display for ele in datalist: t = getTime() savepath, out_img = SetOutputPath(ele, t) p = 0 try: driver, TRYCNT = 0, cm.TRYCNT while driver == 0 and TRYCNT != 0: print("try to setup tbb:", str(TRYCNT)) args = (cm.driverpath, controller, ele[2]) if browser == 'TBB' else () options = { 'TBB': TBBSetup, 'FF': FFSetup, 'CR': ChromeSetup } driver = options[browser](*args) TRYCNT -= 1 if driver == 0: raise TBBSetupError cmd = "tcpdump -i %s tcp and not port ssh -w %s" % ( cm.netInterface, savepath) print('cmd = ', cmd) cmd = cmd.split(' ') p = subprocess.Popen(cmd) try: timeout(cm.VISITPAGE_TIMEOUT) driver.get('https://' + ele[0]) cancel_timeout() time.sleep(cm.DURATION_VISIT_PAGE) p.terminate() if (ele[2] == 0 or ele[2] == 2): driver.get_screenshot_as_file(out_img) writeLog(str(t) + "," + ele[0] + "," + str(ele[2])) print("Finish tcpdump sleep...") except TimeExceededError: writeLog("Error crawling," + ele[0] + "," + str(ele[2]) + "\n" + str("Page visit Timeout")) finally: cancel_timeout() except TBBSetupError: print("[crawl.py error]: unable to setup TBB") writeLog("[crawl.py error]: unable to setup TBB") except Exception as e: with open(cm.ErrorFilePath, 'a+') as fw: fw.write(ele[0] + "," + str(e) + "\n") writeLog("Error crawling," + ele[0] + "," + str(ele[2]) + "\n" + str(e)) finally: if p != 0 and p.returncode != 0: try: p.terminate() except Exception as e: writeLog("[crawl.py] tcpdump terminate error: " + str(e)) if controller != 0: cleanupStream(controller, str(ele[2]), ele[0]) if driver != 0: try: timeout(30) driver.quit() cancel_timeout() except Exception as e: cancel_timeout() writeLog("[crawl.py] driver quit error: " + str(e)) if ele[2] != 3: time.sleep(cm.PAUSE_BETWEEN_INSTANCES) else: time.sleep(cm.PAUSE_BETWEEN_SITES) RemoveTmpFile() RemoveProcess() except TorSetupError: print("[crawl.py] unable to set up tor proxy") writeLog("[crawl.py] unable to set up tor proxy") except Exception as e: print("[crawl.py]launch_tor_with_custom_stem Error") print("Error:", str(e)) writeLog("[crawl.py]launch_tor_with_custom_stem Error : " + str(e)) finally: if tor_process != 0: tor_process.kill() stop_xvfb(xvfb_display)