def cleanup_visit(self):
"""Kill sniffer and Tor browser if they're running."""
wl_log.info("Cleaning up visit.")
wl_log.info("Cancelling timeout")
ut.cancel_timeout()
if self.sniffer and self.sniffer.is_recording:
wl_log.info("Stopping sniffer...")
self.sniffer.stop_capture()
# remove non-tor traffic
self.filter_guards_from_pcap()
if self.tb_driver and self.tb_driver.is_running:
# shutil.rmtree(self.tb_driver.prof_dir_path)
wl_log.info("Quitting selenium driver...")
self.tb_driver.quit()
# close all open streams to prevent pollution
self.tor_controller.close_all_streams()
if self.xvfb and not cm.running_in_CI:
wl_log.info("Stopping display...")
self.vdisplay.stop()
# after closing driver and stoping sniffer, we run postcrawl
self.post_crawl()
def test_cancel_timeout(self):
ut.timeout(1)
ut.cancel_timeout()
try:
sleep(1.1)
except ut.TimeExceededError:
self.fail("Cannot cancel timeout")
def test_cancel_timeout(self):
start_time = time()
ut.timeout(1)
sleep(0.3)
ut.cancel_timeout()
elapsed_time = time() - start_time
self.assertLess(elapsed_time, 5, 'Cancel time does not work. %s' % elapsed_time)
def test_cancel_timeout(self):
start_time = time()
ut.timeout(1)
sleep(0.3)
ut.cancel_timeout()
elapsed_time = time() - start_time
self.assertLess(elapsed_time, 5,
'Cancel time does not work. %s' % elapsed_time)
def close_all_streams(self):
"""Close all streams of a controller."""
print("Closing all streams")
try:
ut.timeout(cm.STREAM_CLOSE_TIMEOUT)
for stream in self.controller.get_streams():
print("Closing stream %s %s %s " %
(stream.id, stream.purpose, stream.target_address))
self.controller.close_stream(stream.id) # MISC reason
except ut.TimeExceededError:
print("Closing streams timed out!")
except:
print("Exception closing stream")
finally:
ut.cancel_timeout()
def close_all_streams(self):
"""Close all streams of a controller."""
wl_log.debug("Closing all streams")
try:
ut.timeout(cm.STREAM_CLOSE_TIMEOUT)
for stream in self.controller.get_streams():
wl_log.debug(
"Closing stream %s %s %s " %
(stream.id, stream.purpose, stream.target_address))
self.controller.close_stream(stream.id) # MISC reason
except ut.TimeExceededError:
wl_log.critical("Closing streams timed out!")
except:
wl_log.debug("Exception closing stream")
finally:
ut.cancel_timeout()
def cleanup_visit(self):
"""Kill sniffer and Tor browser if they're running."""
wl_log.info("Cleaning up visit.")
wl_log.info("Cancelling timeout")
ut.cancel_timeout()
if self.sniffer and self.sniffer.is_recording:
wl_log.info("Stopping sniffer...")
self.sniffer.stop_capture()
if self.tb_driver and self.tb_driver.is_running:
# shutil.rmtree(self.tb_driver.prof_dir_path)
wl_log.info("Quitting selenium driver...")
self.tb_driver.quit()
# close all open streams to prevent pollution
self.tor_controller.close_all_streams()
if self.xvfb and not cm.running_in_CI:
self.vdisplay.stop()
def launch_tor_with_custom_stem(datalist, browser):
print("length of data: ", len(datalist))
tor_binary = join(cm.TorProxypath, cm.DEFAULT_TOR_BINARY_PATH)
tor_process, controller = 0, 0
try:
TRYTOR_CNT = cm.TRYCNT
while TRYTOR_CNT > 0 and tor_process == 0 and controller == 0:
print("try to setup tor:", str(TRYTOR_CNT))
tor_process, controller = TorSetup(tor_binary)
TRYTOR_CNT -= 1
if tor_process == 0:
raise TorSetupError
print("finish tor proxy setup...")
xvfb_display = start_xvfb() # virtual display
for ele in datalist:
t = getTime()
savepath, out_img = SetOutputPath(ele, t)
p = 0
try:
driver, TRYCNT = 0, cm.TRYCNT
while driver == 0 and TRYCNT != 0:
print("try to setup tbb:", str(TRYCNT))
args = (cm.driverpath, controller,
ele[2]) if browser == 'TBB' else ()
options = {
'TBB': TBBSetup,
'FF': FFSetup,
'CR': ChromeSetup
}
driver = options[browser](*args)
TRYCNT -= 1
if driver == 0:
raise TBBSetupError
cmd = "tcpdump -i %s tcp and not port ssh -w %s" % (
cm.netInterface, savepath)
print('cmd = ', cmd)
cmd = cmd.split(' ')
p = subprocess.Popen(cmd)
try:
timeout(cm.VISITPAGE_TIMEOUT)
driver.get('https://' + ele[0])
cancel_timeout()
time.sleep(cm.DURATION_VISIT_PAGE)
p.terminate()
if (ele[2] == 0 or ele[2] == 2):
driver.get_screenshot_as_file(out_img)
writeLog(str(t) + "," + ele[0] + "," + str(ele[2]))
print("Finish tcpdump sleep...")
except TimeExceededError:
writeLog("Error crawling," + ele[0] + "," + str(ele[2]) +
"\n" + str("Page visit Timeout"))
finally:
cancel_timeout()
except TBBSetupError:
print("[crawl.py error]: unable to setup TBB")
writeLog("[crawl.py error]: unable to setup TBB")
except Exception as e:
with open(cm.ErrorFilePath, 'a+') as fw:
fw.write(ele[0] + "," + str(e) + "\n")
writeLog("Error crawling," + ele[0] + "," + str(ele[2]) +
"\n" + str(e))
finally:
if p != 0 and p.returncode != 0:
try:
p.terminate()
except Exception as e:
writeLog("[crawl.py] tcpdump terminate error: " +
str(e))
if controller != 0:
cleanupStream(controller, str(ele[2]), ele[0])
if driver != 0:
try:
timeout(30)
driver.quit()
cancel_timeout()
except Exception as e:
cancel_timeout()
writeLog("[crawl.py] driver quit error: " + str(e))
if ele[2] != 3:
time.sleep(cm.PAUSE_BETWEEN_INSTANCES)
else:
time.sleep(cm.PAUSE_BETWEEN_SITES)
RemoveTmpFile()
RemoveProcess()
except TorSetupError:
print("[crawl.py] unable to set up tor proxy")
writeLog("[crawl.py] unable to set up tor proxy")
except Exception as e:
print("[crawl.py]launch_tor_with_custom_stem Error")
print("Error:", str(e))
writeLog("[crawl.py]launch_tor_with_custom_stem Error : " + str(e))
finally:
if tor_process != 0:
tor_process.kill()
stop_xvfb(xvfb_display)
