def remove_visit_files(visit_info): rm(visit_info.ff_log) rm(visit_info.sys_log) tries = 0 MAX_LOCK_CHECK = 30 rm(visit_info.http_dump) if os.path.isdir(visit_info.profile_dir): # lock file means firefox is still running or didn't exit properly while os.path.isfile(join(visit_info.profile_dir, "lock")): time.sleep(1) tries += 1 if tries > MAX_LOCK_CHECK: break if tries: cm.print_debug(visit_info, "Waited %s sec for the lock to be removed" % tries) shutil.rmtree(visit_info.profile_dir, ignore_errors=True)
def visit_page(url_tuple, timeout=cm.HARD_TIME_OUT, wait_on_site=cm.WAIT_ON_SITE, pre_crawl_sleep=False, out_dir=cm.BASE_TMP_DIR, flash_support=cm.FLASH_ENABLE, cookie_support=cm.COOKIE_ALLOW_ALL): driver = None visit_info = cm.VisitInfo() try: visit_info.rank, visit_info.url = url_tuple except: # When rank of the page is not provided, we'll use rank=0 visit_info.rank, visit_info.url = 0, url_tuple visit_info.sys_log = join( out_dir, "syscall-%s-%s.log" % (visit_info.rank, ut.rand_str())) visit_info.http_log = join( out_dir, "http-%s-%s.log" % (visit_info.rank, ut.rand_str())) visit_info.http_dump = join( out_dir, "mitm-%s-%s.dmp" % (visit_info.rank, ut.rand_str())) visit_info.start_time = strftime("%Y%m%d-%H%M%S") visit_info.out_dir = out_dir visit_info.out_db = join(visit_info.out_dir, cm.DB_FILENAME) visit_info.err_log = join(out_dir, "error.log") visit_info.debug_log = join(out_dir, "debug.log") be = cm.BrowserEvent() be.event_type = cm.EVENT_NEW_VISIT visit_info.ff_log = open_log_file(out_dir, visit_info.url) if not visit_info.url[:5] in ('data:', 'http:', 'https', 'file:'): visit_info.url = 'http://' + visit_info.url try: visit_info.visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be, visit_info) cm.print_debug( visit_info, "Visiting: %s %s (%s)" % (visit_info.visit_id, visit_info.url, visit_info.rank)) setup_nspr_logging(visit_info.http_log) visit_info.vdisplay = start_xvfb() port, visit_info.mitm_proc = start_mitm_capture(visit_info.http_dump) driver, visit_info.profile_dir, visit_info.sel_proc =\ get_browser(visit_info.ff_log, port, flash_support, cookie_support) if flash_support: visit_info.strace_proc = log_syscalls(visit_info.sel_proc, visit_info.sys_log) ############################################################# driver_get(driver, visit_info, cm.SOFT_TIMEOUT) # real visit ############################################################# time.sleep(wait_on_site) close_driver(driver, timeout=10) stop_strace(visit_info.strace_proc) result_dict = process_crawler_output(visit_info.ff_log, visit_info, flash_support) cm.print_debug( visit_info, "Visit OK: %s %s (%s)" % (visit_info.visit_id, visit_info.url, visit_info.rank)) visit_info.incomplete = 0 dbu.insert_to_db(dbu.DBCmd.UPDATE_VISIT, be, visit_info) quit_driver(driver) stop_xvfb(visit_info.vdisplay) remove_visit_files(visit_info) except (cm.TimeExceededError, sel_exceptions.TimeoutException) as texc: err_str = "Visit to %s(%s) timed out %s" % \ (visit_info.url, visit_info.rank, texc) cm.print_error(visit_info, err_str) clean_up(visit_info, driver) return None except Exception as exc: err_str = "Exception visiting %s(%s) %s %s" % \ (visit_info.url, visit_info.rank, exc, traceback.format_exc()) cm.print_error(visit_info, err_str) clean_up(visit_info, driver) return None else: return result_dict
def visit_page(url_tuple, timeout=cm.HARD_TIME_OUT, wait_on_site=cm.WAIT_ON_SITE, pre_crawl_sleep=False, out_dir=cm.BASE_TMP_DIR, flash_support=cm.FLASH_ENABLE, cookie_support=cm.COOKIE_ALLOW_ALL): driver = None visit_info = cm.VisitInfo() try: visit_info.rank, visit_info.url = url_tuple except: # When rank of the page is not provided, we'll use rank=0 visit_info.rank, visit_info.url = 0, url_tuple visit_info.sys_log = join(out_dir, "syscall-%s-%s.log" % (visit_info.rank, ut.rand_str())) visit_info.http_log = join(out_dir, "http-%s-%s.log" % (visit_info.rank, ut.rand_str())) visit_info.http_dump = join(out_dir, "mitm-%s-%s.dmp" % (visit_info.rank, ut.rand_str())) visit_info.start_time = strftime("%Y%m%d-%H%M%S") visit_info.out_dir = out_dir visit_info.out_db = join(visit_info.out_dir, cm.DB_FILENAME) visit_info.err_log = join(out_dir, "error.log") visit_info.debug_log = join(out_dir, "debug.log") be = cm.BrowserEvent() be.event_type = cm.EVENT_NEW_VISIT visit_info.ff_log = open_log_file(out_dir, visit_info.url) if not visit_info.url[:5] in ('data:', 'http:', 'https', 'file:'): visit_info.url = 'http://' + visit_info.url try: visit_info.visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be, visit_info) cm.print_debug(visit_info, "Visiting: %s %s (%s)" % (visit_info.visit_id, visit_info.url, visit_info.rank)) setup_nspr_logging(visit_info.http_log) visit_info.vdisplay = start_xvfb() port, visit_info.mitm_proc = start_mitm_capture(visit_info.http_dump) driver, visit_info.profile_dir, visit_info.sel_proc =\ get_browser(visit_info.ff_log, port, flash_support, cookie_support) if flash_support: visit_info.strace_proc = log_syscalls(visit_info.sel_proc, visit_info.sys_log) ############################################################# driver_get(driver, visit_info, cm.SOFT_TIMEOUT) # real visit ############################################################# time.sleep(wait_on_site) close_driver(driver, timeout=10) stop_strace(visit_info.strace_proc) result_dict = process_crawler_output(visit_info.ff_log, visit_info, flash_support) cm.print_debug(visit_info, "Visit OK: %s %s (%s)" % (visit_info.visit_id, visit_info.url, visit_info.rank)) visit_info.incomplete = 0 dbu.insert_to_db(dbu.DBCmd.UPDATE_VISIT, be, visit_info) quit_driver(driver) stop_xvfb(visit_info.vdisplay) remove_visit_files(visit_info) except (cm.TimeExceededError, sel_exceptions.TimeoutException) as texc: err_str = "Visit to %s(%s) timed out %s" % \ (visit_info.url, visit_info.rank, texc) cm.print_error(visit_info, err_str) clean_up(visit_info, driver) return None except Exception as exc: err_str = "Exception visiting %s(%s) %s %s" % \ (visit_info.url, visit_info.rank, exc, traceback.format_exc()) cm.print_error(visit_info, err_str) clean_up(visit_info, driver) return None else: return result_dict