def execute(self, url): """ Creates a selenium webbrowser-instance( with the profile and options set in the attributes) and retrieves Data from Website :param url: The Url ist the Target on which the included actions will be performed on The results will be stored in class attributes """ """Creation of Webbrowser instance""" profile = FirefoxProfile(self.profilepath) profile.set_preference("network.cookie.cookieBehavior", 0) profile.update_preferences() options = Options() options.headless = self.headless browser = webdriver.Firefox(firefox_profile=profile, options=options, executable_path=self.geckopath) browser.get(url) """Retrieval of Data""" """Get Sourcecode""" self.source = browser.page_source """Get the currently used URL""" self.trueurl = browser.current_url """Get Cookies from the opened website""" self.cookies = browser.get_cookies() """make screenshots of the page""" time.sleep(3) self.imagepath = str(os.path.abspath( os.getcwd())) + '\screenshot' + str(self)[-19:-1] + '.png' browser.save_screenshot(self.imagepath) """quit Browser to clear temporary storage""" browser.quit()
def init_glob_vars(): global PROFILE_glob global CAPABILITIES_glob global SCRIPT_DIR_glob global EXECUTABLE_PATH_glob global PROXY_ADDRESS_glob global PROXY_glob # Browser settings PROFILE_glob = FirefoxProfile( '/Users/altay.amanbay/Library/Application Support/Firefox/Profiles') PROFILE_glob.set_preference("network.proxy.type", 1) PROFILE_glob.set_preference("network.proxy.socks", "127.0.0.1") PROFILE_glob.set_preference("network.proxy.socks_port", 9150) PROFILE_glob.set_preference("network.proxy.socks_version", 5) PROFILE_glob.set_preference("network.proxy.socks_remote_dns", True) PROFILE_glob.update_preferences() CAPABILITIES_glob = DesiredCapabilities.FIREFOX CAPABILITIES_glob["marionette"] = True CAPABILITIES_glob[ "binary"] = "/Applications/Firefox.app/Contents/MacOS/firefox-bin" # Proxy (start Tor browser before executing script) PROXY_ADDRESS_glob = "127.0.0.1:9150" # localhost and Tor browser's default port number PROXY_glob = Proxy({ 'proxyType': ProxyType.MANUAL, 'httpProxy': PROXY_ADDRESS_glob, }) # Webdriver path SCRIPT_DIR_glob = os.path.dirname(os.path.abspath(__file__)) print('Abs path:', SCRIPT_DIR_glob) EXECUTABLE_PATH_glob = SCRIPT_DIR_glob + '/webdrivers/geckodriver - v0.18.0/geckodriver'
def prepareTor(): if not Settings['TOR']: profile = FirefoxProfile() else: profile = ( FirefoxProfile(Settings['PROFILE_PATH']) if Settings['PROFILE_PATH'] else FirefoxProfile() ) profile.set_preference('network.proxy.type', 1) profile.set_preference('network.proxy.socks', '127.0.0.1') profile.set_preference('network.proxy.socks_port', 9050) profile.set_preference('network.proxy.socks_remote_dns', False) random.shuffle(Settings['USER_AGENTS']) profile.set_preference("general.useragent.override", Settings['USER_AGENTS'][0]) profile.set_preference("intl.accept_languages", "en-US") profile.update_preferences() driver = webdriver.Firefox(firefox_profile = profile, executable_path=Settings['GECKO_DRIVER']) driver.get("http://check.torproject.org") driver.implicitly_wait(10) return driver
def session_create(config): log.info("Creating session") options = Options() if config.get('headless', False) is True: log.info("Headless mode") options.add_argument("--headless") if config.get('webdriver_enabled') is False: options.set_preference("dom.webdriver.enabled", False) # selenium_profile = webdriver.FirefoxProfile('/home/container-dev/.mozilla/firefox/') # selenium_profile.update_preferences() # options.add_argument("-profile") # options.add_argument("/tmp/ff1") # driver = webdriver.Firefox(options=options, service_log_path=path.join("..", "data","geckodriver.log")) # driver = webdriver.Chrome() profile = FirefoxProfile() profile.set_preference("dom.webdriver.enabled", False) profile.set_preference('useAutomationExtension', False) profile.update_preferences() desired = DesiredCapabilities.FIREFOX driver = webdriver.Firefox(options=options, firefox_profile=profile, desired_capabilities=desired) log.info("New session is: %s %s" % (driver.session_id, driver.command_executor._url)) return driver
def get_firefox_profile(cls, temp_download_dir): profile = FirefoxProfile() profile.set_preference("browser.download.folderList", 2) profile.set_preference("browser.download.dir", str(temp_download_dir.absolute())) profile.set_preference("browser.download.useDownloadDir", True) profile.set_preference( "browser.download.viewableInternally.enabledTypes", "") profile.set_preference( "browser.helperApps.neverAsk.saveToDisk", "application/pdf;text/plain;application/text;text/xml;application/xml;application/vnd.openxmlformats-officedocument.wordprocessingml.document;application/rtf;application/vnd.rar;text/plain;image/webp;image/bmp;image/jpeg;application/x-7z-compressed;application/zip;application/x-tar;application/gzip;application/vnd.openxmlformats-officedocument.presentationml.presentation;application/octet-stream;application/x-rar-compressed;application/x-zip-compressed;multipart/x-zip;application/vnd.oasis.opendocument.presentation;application/vnd.oasis.opendocument.spreadsheet;application/vnd.oasis.opendocument.text", ) profile.set_preference("pdfjs.disabled", True) profile.update_preferences() return profile
def setUp(self): profile = FirefoxProfile() # Ignore certification profile.set_preference("webdriver_assume_untrusted_issuer", False) profile.set_preference("webdriver_accept_untrusted_certs", True) profile.accept_untrusted_certs = True # Set user agent profile.set_preference("general.useragent.override", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A365 Safari/600.1.4") profile.update_preferences() self.driver = webdriver.Firefox(profile) self.base_url = "http://m.finance.yahoo.co.jp/" self.driver.implicitly_wait(30) self.accept_next_alert = True
def br(self): display = Display(visible=0, size=display_size) display.start() profile = FirefoxProfile() profile.set_preference('network.proxy.type', 1) profile.set_preference('network.proxy.socks', tor_proxy) profile.set_preference('network.proxy.socks_port', tor_port) profile.set_preference("network.proxy.socks_remote_dns", False) profile.update_preferences() driver = webdriver.Firefox(firefox_profile=profile, executable_path=geckodriver_path) # driver = webdriver.Firefox(executable_path = geckodriver_path) with self.lock: self.browslist.append(driver)
def _load_firefox_profile(): # create a firefox profile using the template in data/firefox_profile.js.template global firefox_profile_tmpdir if firefox_profile_tmpdir is None: firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_') # Clean up tempdir at exit atexit.register(rmtree, firefox_profile_tmpdir) template = data_path.join('firefox_profile.js.template').read() profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir) profile_dict = json.loads(profile_json) profile = FirefoxProfile(firefox_profile_tmpdir) for pref in profile_dict.iteritems(): profile.set_preference(*pref) profile.update_preferences() return profile
def _load_firefox_profile(): # create a firefox profile using the template in data/firefox_profile.js.template # Make a new firefox profile dir if it's unset or doesn't exist for some reason firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_') log.debug("created firefox profile") # Clean up tempdir at exit atexit.register(rmtree, firefox_profile_tmpdir, ignore_errors=True) template = data_path.join('firefox_profile.js.template').read() profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir) profile_dict = json.loads(profile_json) profile = FirefoxProfile(firefox_profile_tmpdir) [profile.set_preference(*pref) for pref in profile_dict.items()] profile.update_preferences() return profile
def get_driver(device): fp = FirefoxProfile() # fp.set_preference('media.navigator.permission.disabled', True) fp.update_preferences() caps = DesiredCapabilities.FIREFOX.copy() caps['firefox_profile'] = fp.encoded # caps['marionette'] = True # webdriver.Firefox(capabilities=caps) # options = Options() # options.set_headless(True) # options.add_argument('--hide-scrollbars') # caps = options.to_capabilities() driver = webdriver.Remote( command_executor=settings.SELENIUM_COMMAND_EXECUTER, desired_capabilities=caps, ) return driver
def setUp(self): profile = FirefoxProfile() # Ignore certification profile.set_preference("webdriver_assume_untrusted_issuer", False) profile.set_preference("webdriver_accept_untrusted_certs", True) profile.accept_untrusted_certs = True # Set user agent profile.set_preference( "general.useragent.override", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A365 Safari/600.1.4" ) profile.update_preferences() self.driver = webdriver.Firefox(profile) self.base_url = "http://m.finance.yahoo.co.jp/" self.driver.implicitly_wait(30) self.accept_next_alert = True
def _load_firefox_profile(): # create a firefox profile using the template in data/firefox_profile.js.template global firefox_profile_tmpdir if firefox_profile_tmpdir is None: firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_') # Clean up tempdir at exit atexit.register(rmtree, firefox_profile_tmpdir) template = data_path.join('firefox_profile.js.template').read() profile_json = Template(template).substitute( profile_dir=firefox_profile_tmpdir) profile_dict = json.loads(profile_json) profile = FirefoxProfile(firefox_profile_tmpdir) for pref in profile_dict.iteritems(): profile.set_preference(*pref) profile.update_preferences() return profile
def _load_firefox_profile(): # create a firefox profile using the template in data/firefox_profile.js.template # Make a new firefox profile dir if it's unset or doesn't exist for some reason firefox_profile_tmpdir = mkdtemp(prefix='firefox_profile_') log.debug("created firefox profile") # Clean up tempdir at exit atexit.register(rmtree, firefox_profile_tmpdir, ignore_errors=True) template = data_path.join('firefox_profile.js.template').read() profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir) profile_dict = json.loads(profile_json) profile = FirefoxProfile(firefox_profile_tmpdir) for pref in profile_dict.items(): profile.set_preference(*pref) profile.update_preferences() return profile
def create_firefox_ninja(self, desired_capabilities, profile, options, ip): """ Recommended to edit chromedriver.exe by replacing all 'cdc_' text to 'dog_' or anything else. Also, `profile` argument is ignored. """ if profile is not None: self.log.warning( 'Firefox ninja received a `profile` which will be ' 'ignored and replaced by `ninja profile`.') profile = FirefoxProfile() profile.set_preference("dom.webdriver.enabled", False) profile.set_preference('useAutomationExtension', False) profile.update_preferences() if desired_capabilities is None: desired_capabilities = DesiredCapabilities.FIREFOX desired_capabilities['marionette'] = True return self.create_firefox(desired_capabilities, profile, options, ip)
def _load_firefox_profile(): # create a firefox profile using the template in data/firefox_profile.js.template global firefox_profile_tmpdir # Make a new firefox profile dir if it's unset or doesn't exist for some reason if firefox_profile_tmpdir is None or not os.path.exists(firefox_profile_tmpdir): firefox_profile_tmpdir = mkdtemp(prefix="firefox_profile_") # Clean up tempdir at exit atexit.register(rmtree, firefox_profile_tmpdir, ignore_errors=True) template = data_path.join("firefox_profile.js.template").read() profile_json = Template(template).substitute(profile_dir=firefox_profile_tmpdir) profile_dict = json.loads(profile_json) profile = FirefoxProfile(firefox_profile_tmpdir) for pref in profile_dict.iteritems(): profile.set_preference(*pref) profile.update_preferences() return profile
def connect(pathTor, pathProfile, isHeadless): torexe = os.popen(pathTor) profile = FirefoxProfile(pathProfile) profile.set_preference('network.proxy.type', 1) profile.set_preference('network.proxy.socks', '127.0.0.1') profile.set_preference('network.proxy.socks_port', 9050) profile.set_preference("network.proxy.socks_remote_dns", False) profile.update_preferences() if isHeadless: options = Options() options.add_argument('--headless') options.add_argument( '--disable-gpu') # Last I checked this was necessary. return webdriver.Firefox(firefox_profile=profile, executable_path=r'geckodriver.exe', options=options) else: return webdriver.Firefox(firefox_profile=profile, executable_path=r'geckodriver.exe')
def init(self, config): self.config = config print self.config.browser_driver_command_executor command_executor = self.config.browser_driver_command_executor if self.config.browser_profile_dir: profile = FirefoxProfile( profile_directory=self.config.browser_profile_dir) self.dmesg('loading browser profile') else: profile = None self.dmesg('browser profile not configured, use None profile') self.dmesg('trying start a browser') px = config.proxy_type if px in ['socks', 'http']: self.dmesg('apply %s proxy, %s:%s' % (px, config.proxy_host, config.proxy_port)) profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.%s" % px, config.proxy_host) profile.set_preference("network.proxy.%s_port" % px, int(config.proxy_port)) profile.set_preference("network.proxy.socks_version", 5) profile.update_preferences() self.driver = webdriver.Remote( command_executor=self.config.browser_driver_command_executor, desired_capabilities=DesiredCapabilities.FIREFOX, browser_profile=profile) try: self.driver.get(self.config.start_url) except: self.dmesg('打开失败,10s 后重试...') sleep(10) try: self.driver.get(self.config.start_url) except: self.dmesg('打开失败,10s 后重试...') sleep(10) self.driver.get(self.config.start_url)
def __create_driver(): """ Метод создания дрйавера с заданными настройками """ binary = FirefoxBinary(FIREFOX_BINARY) options = Options() if HEADLESS: options.add_argument("--headless") if USE_TOR: os.popen(TOR_EXE) profile = FirefoxProfile(TOR_PROFILE) else: profile = FirefoxProfile() if USE_TOR or USE_PROXY: profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.socks", PROXY_HOST) profile.set_preference("network.proxy.socks_port", PROXY_PORT) profile.set_preference("network.proxy.socks_remote_dns", False) if not IMAGES: profile.set_preference('permissions.default.image', 2) if not CSS: profile.set_preference('permissions.default.stylesheet', 2) if not FLASH: profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') if not NOTIFICATIONS: profile.set_preference("dom.push.enabled", False) profile.update_preferences() return webdriver.Firefox(firefox_profile=profile, firefox_binary=binary, options=options)
class antidriver: def __init__(self): pass def chrome(self): option = webdriver.ChromeOptions() option.add_argument( '--user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"' ) option.add_experimental_option('excludeSwitches', [ 'enable-automation', "ignore-certificate-errors", "safebrowsing-disable-download-protection", "safebrowsing-disable-auto-update", "disable-client-side-phishing-detection" ]) #option.add_argument('--headless') #username = os.getenv("USERNAME") #userProfile = "C:\\Users\\" + username + "\\AppData\\Local\\Google\\Chrome\\User Data\\Default" #option.add_argument("user-data-dir={}".format(userProfile)) option.add_argument('disable-infobars') localpath = "Google Chrome\chromedriver.exe" os.environ["webdriver.chrome.driver"] = localpath self.driver = webdriver.Chrome(chrome_options=option, executable_path=localpath) self.driver.delete_all_cookies() return self.driver def firefox(self, data={"ip": "221.225.91.175", "port": 43301}): '以代理的形式启动一个firefox' self.location = 'C:\Program Files\Mozilla Firefox\\firefox.exe' self.profile = FirefoxProfile() self.profile.set_preference("network.proxy.type", 1) self.profile.set_preference('network.proxy.http', data['ip']) self.profile.set_preference('network.proxy.http_port', data['port']) # int self.profile.update_preferences() self.driver = webdriver.Firefox(firefox_profile=self.profile, firefox_binary=self.location) return self.driver
def get_firefox_webdriver_args(self): options = selenium.webdriver.FirefoxOptions() if self['headless']: options.add_argument('-headless') profile = FirefoxProfile( os.environ.get( 'GOCEPT_WEBDRIVER_FF_PROFILE', os.environ.get('GOCEPT_SELENIUM_FF_PROFILE'))) profile.native_events_enabled = True profile.update_preferences() # Save downloads always to disk into a predefined dir. profile.set_preference("browser.download.folderList", 2) profile.set_preference( "browser.download.manager.showWhenStarting", False) profile.set_preference( "browser.download.dir", str(self['selenium_download_dir'])) profile.set_preference( "browser.helperApps.neverAsk.saveToDisk", "application/pdf") profile.set_preference("pdfjs.disabled", True) return {'options': options, 'firefox_profile': profile}
def firefox_builder(): binary = FirefoxBinary(FIREFOX_BINARY_PATH) profile = FirefoxProfile() # Flash settings # if platform.system().lower() != "windows": # profile.set_preference("plugin.flash.path", FLASH_PATH) # profile.set_preference("dom.ipc.plugins.flash.disable-protected-mode", True) # profile.set_preference("plugins.flashBlock.enabled", False) # profile.set_preference("plugin.state.flash", 2) profile.set_preference( "general.useragent.override", "UserAgent: Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" ) profile.set_preference("app.update.auto", False) profile.set_preference("app.update.enabled", False) # apply the setting under (A) to ALL new windows (even script windows with features) profile.set_preference("browser.link.open_newwindow.restriction", 0) profile.set_preference("browser.link.open_newwindow.override.external", 2) # open external links in a new window profile.set_preference("browser.link.open_newwindow", 3) # divert new window to a new tab ## # profile.set_preference("network.http.connection-timeout", 15) # profile.set_preference("network.http.connection-retry-timeout", 15) ## profile.update_preferences() opts = FirefoxOptions opts.headless = HEADLESS browser = webdriver.Firefox(firefox_binary=binary, options=opts, firefox_profile=profile, executable_path=FIREFOX_DRIVER_PATH) # firefox flash check # browser.get("about:config") # browser.find_element_by_xpath('//*[@id="warningButton"]').click() # browser.find_element_by_css_selector( # "window#config deck#configDeck vbox hbox#filterRow textbox#textbox input.textbox-input").send_keys( # "flash" + Keys.ENTER) # browser.save_screenshot("sc.png") # logging.info("about:config => flash settings, screenshot captured") # sleep(1) # browser.get("https://isflashinstalled.com/") # logging.info(f"{browser.find_element_by_css_selector('body').text.split()[:4]}") # browser.get("https://www.whatismybrowser.com/detect/is-flash-installed") # is_installed = re.search("Flash \d\d?.\d\d?.\d\d? is installed", # browser.find_element_by_xpath('//*[@id="detected_value"]').text) # browser.implicitly_wait(10) # browser.get("https://toolster.net/flash_checker") # elmnt = browser.find_element_by_css_selector( # "html body div#main div#center div#tool_padding div#flash_checker div#bottom_info div#double-version.vtor_info") # is_installed = re.search(r"You have installed Flash Player v.\d\d?.\d\d?.\d\d?", elmnt.text) # # assert is_installed, "Flash is disabled or not installed!" # logging.info(f"Check flash response: {is_installed.group()}") return browser
def create_profile(path=None, user_agent=None, accept_language=None, proxy=None, proxy_type=None, no_proxy_hosts=None, download_directory=None, download_content_type=None): """ @paramDownload_content_type: CSV string """ if path is not None: profile = FirefoxProfile(path) else: profile = FirefoxProfile() # Memory and cpu optimization profile.set_preference('browser.sessionhistory.max_total_viewers', 0) #profile.set_preference('browser.cache.memory.enable', False) #profile.set_preference('browser.cache.offline.enable', False) #profile.set_preference('browser.cache.disk.enable', False) profile.set_preference('browser.safebrowsing.enabled', False) profile.set_preference('browser.shell.checkDefaultBrowser', False) profile.set_preference('browser.startup.page', 0) profile.set_preference('dom.ipc.plugins.enabled.timeoutSecs', 15) profile.set_preference('dom.max_script_run_time', 10) profile.set_preference('extensions.checkCompatibility', False) profile.set_preference('extensions.checkUpdateSecurity', False) profile.set_preference('extensions.update.autoUpdateEnabled', False) profile.set_preference('extensions.update.enabled', False) profile.set_preference('network.http.max-connections-per-server', 30) profile.set_preference('network.prefetch-next', False) profile.set_preference('plugin.default_plugin_disabled', False) profile.set_preference('print.postscript.enabled', False) profile.set_preference('toolkit.storage.synchronous', 0) profile.set_preference('image.animation_mode', 'none') profile.set_preference('images.dither', False) profile.set_preference('content.notify.interval', 1000000) profile.set_preference('content.switch.treshold', 100000) profile.set_preference('nglayout.initialpaint.delay', 1000000) profile.set_preference('network.dnscacheentries', 200) profile.set_preference('network.dnscacheexpiration', 600) if user_agent is not None: profile.set_preference("general.useragent.override", user_agent) if accept_language is not None: profile.set_preference('intl.accept_languages', accept_language) if proxy is not None: logging.debug('Setting up proxy %s [%s]' % (proxy, proxy_type)) server, port = proxy.split(':') if proxy_type == 'socks5': profile.set_preference("network.proxy.socks", server) profile.set_preference("network.proxy.socks_port", int(port)) elif proxy_type == 'http': profile.set_preference("network.proxy.http", server) profile.set_preference("network.proxy.http_port", int(port)) else: raise Exception('Unkown proxy type: %s' % proxy_type) profile.set_preference("network.proxy.type", 1) if no_proxy_hosts is not None: csv = ', '.join(no_proxy_hosts) profile.set_preference('network.proxy.no_proxies_on', 'localhost, 127.0.0.1, %s' % csv) if download_directory is not None and download_content_type is not None: profile.set_preference("browser.download.folderList", 2) profile.set_preference("browser.download.manager.showWhenStarting", False) profile.set_preference("browser.download.dir", download_directory) profile.set_preference("browser.helperApps.neverAsk.saveToDisk", download_content_type) profile.update_preferences() return profile
f = open(os.getcwd() + '\\SBR_MLB_Lines_' + season + '_line_moves.txt', 'a') f.write('Date,Team,Team_SP,Opp,Opp_SP,Bet_Length,Bet_Type,Line_Move_Time,Over_Under,Line,Odds') f.write('\n') f.close() ## from selenium import webdriver from selenium.webdriver.firefox.firefox_profile import FirefoxProfile import os # torexe = os.popen(r'/home/kc/Downloads/tor-browser_en-US/Browser/TorBrowser/Tor/tor.exe') profile = FirefoxProfile(r'/home/kc/Downloads/tor-browser_en-US/Browser/TorBrowser/Data/Browser/profile.default') # profile = FirefoxProfile(r'C:/users/kj/Desktop/torbrowser/Browser/TorBrowser/Data/Browser/profile.default') profile.set_preference('network.proxy.type', 1) profile.set_preference('network.proxy.socks', '127.0.0.1') profile.set_preference('network.proxy.socks_port', 9150) profile.set_preference("network.proxy.socks_remote_dns", False) profile.update_preferences() # driver.get("http://check.torproject.org") # time.sleep(5) # to uitilize a headless driver, download and install phantomjs and use below to open driver instead of above line # download link -- http://phantomjs.org/download.html # driver = webdriver.PhantomJS(r"C:\Users\Monstar\Python\phantomjs-2.0.0\bin\phantomjs.exe") for y in range(int(start_month) - 1, int(end_month)): try: run_main(profile, season, y) except KeyboardInterrupt: break
firefox_options.binary_location = '/Applications/Firefox.app/Contents/MacOS/firefox' # firefox_options.headless = True # FirefoxProfile firefox_profile = FirefoxProfile(); firefox_profile.set_preference('browser.privatebrowsing.autostart', True) firefox_profile.set_preference('pdfjs.disabled', True) firefox_profile.set_preference('browser.download.folderList', 2) firefox_profile.set_preference('browser.download.panel.shown', False) firefox_profile.set_preference('browser.tabs.warnOnClose', False) firefox_profile.set_preference('browser.tabs.animate', False) firefox_profile.set_preference('browser.fullscreen.animateUp', 0) firefox_profile.set_preference('geo.enabled', False) firefox_profile.set_preference('browser.urlbar.suggest.searches', False) firefox_profile.set_preference('browser.tabs.warnOnCloseOtherTabs', False) firefox_profile.update_preferences() # DesiredCapabilities firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX.copy() firefox_capabilities['marionette'] = True firefox_capabilities['acceptInsecureCerts'] = True # iniciar navegador w = webdriver.Firefox(options=firefox_options, firefox_profile=firefox_profile, desired_capabilities=firefox_capabilities) # tempo de espera padrao w.implicitly_wait(15) w.get('https://www.uol.com.br')
def bidCrawling(request): if request.method == 'POST': form = bidCrawlingForm(request.POST) if form.is_valid(): searchStartBid = form.cleaned_data['searchstartbid'] searchEndBid = form.cleaned_data['searchendbid'] bidSearchStart = int(searchStartBid) bidSearchEnd = int(searchEndBid) engine = create_engine( "mysql+pymysql://root:[email protected]:3306/book?charset=utf8mb4", encoding='utf8') torexe = os.popen(r'C:\Dev_program\Tor Browser\Browser\firefox.exe') profile = FirefoxProfile( r'C:\Dev_program\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default' ) profile.set_preference('network.proxy.type', 1) profile.set_preference('network.proxy.socks', '127.0.0.1') profile.set_preference('network.proxy.socks_port', 9050) profile.set_preference("network.proxy.socks_remote_dns", False) profile.update_preferences() driver = webdriver.Firefox( firefox_profile=profile, executable_path=r'C:\Dev_program\geckodriver.exe') driver.implicitly_wait(time_to_wait=5) searchCount = bidSearchEnd - bidSearchStart + 1 searchMok = int(np.ceil((bidSearchEnd - bidSearchStart + 1) / 50)) taskId = 1 while searchMok > 0: title_list = [] intro_list = [] author_intro_list = [] category_top_list = [] category_middle_list = [] category_bottom_list = [] ISBN_list = [] writer_list = [] translator_list = [] painter_list = [] publisher_list = [] publish_date_list = [] content_list = [] bid_list = [] image_list = [] grade_list = [] review_list = [] writer_name = [] writer_link = [] writer_num = [] writer_book_title = [] writer_bid = [] writer_isbn = [] taskId_list = [] taskContent_list = [] str_list = [] end_list = [] complete_list = [] errorDetail_list = [] crawlerNum_list = [] taskId_list.append(taskId) if (bidSearchStart + 49) < bidSearchEnd: taskContent = str(bidSearchStart) + '~' + str(bidSearchStart + 49) else: taskContent = str(bidSearchStart) + '~' + str(bidSearchEnd) taskContent_list.append(taskContent) str_now = datetime.datetime.now() str_time = str_now.strftime('%Y-%m-%d %H:%M:%S') str_list.append(str_time) for i in range(bidSearchStart, bidSearchStart + 50): url_det = 'https://book.naver.com/bookdb/book_detail.nhn?bid=' + \ str(i) try: html_det = urlopen(url_det) except (HTTPError, URLError, IndexError) as e: errorDetail_list.append(e) end_now = datetime.datetime.now() end_time = end_now.strftime('%Y-%m-%d %H:%M:%S') end_list.append(end_time) complete_list.append('error') crawlerNum_list.append(11) task_list = [] for taskId, taskContent, str_time, end_time, complete, errorDetail, crawlerNum in zip( taskId_list, taskContent_list, str_list, end_list, complete_list, errorDetail_list, crawlerNum_list): task = { "taskId": taskId, "taskContent": taskContent, "str_time": str_time, "end_time": end_time, "complete": complete, "errorDetail": errorDetail, "crawlerNum": crawlerNum } task_list.append(task) task_DF = '' task_DF = pd.DataFrame(task_list) conn = engine.connect() task_DF.to_sql(name='task', con=engine, if_exists='append', index=False) conn.close() print(e) print('에러로 크롤링 종료') time.sleep(3) if bidSearchStart + 50 > bidSearchEnd: driver.quit() else: continue else: soup_det = BeautifulSoup(html_det, "html.parser") if '책정보, :' in soup_det.text: print(str(i), "번 제외(삭제 서지)") continue else: pass book_info = soup_det.find('div', class_='book_info_inner') try: book_info_text = book_info.get_text() except AttributeError: print(str(i), "번 제외(Attr 에러)") continue if driver.current_url == 'https://nid.naver.com/nidlogin.login?svctype=128&a_version=2&viewtype=2&url=http://book.naver.com&surl=http://book.naver.com': continue else: pass try: book_intro = soup_det.find('div', id='bookIntroContent') book_intro_text = book_intro.get_text().replace('\n', '') intro_list.append(book_intro_text) except AttributeError: book_intro_text = '' intro_list.append(book_intro_text) try: author_intro = soup_det.find('div', id='authorIntroContent') author_intro_text = author_intro.get_text().replace('\n', '') author_intro_list.append(author_intro_text) except AttributeError: author_intro_text = '' author_intro_list.append(author_intro_text) try: category_top = soup_det.find('li', class_='select') category_top_text = category_top.get_text().replace('\n', '') category_top_list.append(category_top_text) except AttributeError: category_top_text = '' category_top_list.append(category_top_text) try: category_middle = soup_det.find('li', class_='select2') category_middle_text = category_middle.get_text().replace( '\n', '') category_middle_list.append(category_middle_text) except AttributeError: category_middle_text = '' category_middle_list.append(category_middle_text) try: category_bottom = soup_det.find('li', class_='select3') category_bottom_text = category_bottom.get_text().replace( '\n', '') category_bottom_list.append(category_bottom_text) except AttributeError: category_bottom_text = '' category_bottom_list.append(category_bottom_text) try: grade = soup_det.find( "div", class_="txt_desc").find("strong").text[:-1] except AttributeError: grade = '' grade_list.append(grade) try: review = soup_det.find( "a", id="txt_desc_point").find_all("strong")[1].text except AttributeError: review = '' review_list.append(review) bookinfo_line1 = book_info.find_all("div")[2] rel_name = bookinfo_line1.text rel_list = [] for rel in bookinfo_line1.find_all("em"): rel_cate = rel.text rel_list.append(rel_cate) for r in range(0, len(rel_list)): rel_name = rel_name.replace(rel_list[r], '') rel_name = rel_name.split('|') publish_date = rel_name[-1] if len(publish_date) == 4: publish_date = publish_date + ".01.01" elif len(publish_date) == 6: publish_date = publish_date[:4] + "." + publish_date[4:] + ".01" elif publish_date == '': publish_date = '2025.01.01' if publish_date[0] != '1' and publish_date[0] != '2': publish_date = '2025.01.01' publish_date_list.append(publish_date) publisher = rel_name[-2] publisher_list.append(publisher) rel_name = rel_name[1:-2] rel_list = rel_list[1:] if (len(rel_list) and len(rel_name)) == 2: painter = rel_name[0].replace('\n', '') translator = rel_name[1].replace('\n', '') elif (len(rel_list) and len(rel_name)) == 1: if '역자' in rel_list: translator = rel_name[0].replace('\n', '') painter = '' else: translator = '' painter = rel_name[0].replace('\n', '') else: translator = '' painter = '' translator_list.append(translator) painter_list.append(painter) ISBN_str = book_info_text.find('ISBN') + 6 ISBN_end = book_info_text.find('|', ISBN_str) if ISBN_end == -1: ISBN_end = book_info_text.find('\n', ISBN_str) ISBN = book_info_text[ISBN_str:ISBN_end] if '\n' in ISBN: ISBN = ISBN[0:int(ISBN.find('\n'))] ISBN_list.append(ISBN) content = '' content = get_text_list(soup_det.select("div.book_cnt")) if content == []: content = [""] content_list.append(content) else: content_list.append(content) for src in soup_det.find("div", class_="thumb_type").find_all("a"): bookImage = src.find("img")["src"] image_list.append(bookImage) writer_a = soup_det.find( "div", class_="book_info_inner").find_all("div")[2].find_all("a")[:-1] writer_book = soup_det.find("div", class_="book_info").find("a").text writer_book_bid = soup_det.find( "div", class_="book_info").find("a")["href"].split("=")[1] bid_list.append(writer_book_bid) title_list.append(writer_book) writer = soup_det.find("div", class_="book_info_inner").find_all( "div")[2].text.split("|")[0][3:].strip() writer_list.append(writer) for w in range(0, len(writer_a)): writer_n = writer_a[w].text writer_name.append(writer_n) writer_href = writer_a[w]["href"] writer_link.append(writer_href) writer_split = writer_a[w]["href"].split("=")[3] writer_num.append(writer_split) writer_book_title.append(writer_book) writer_bid.append(writer_book_bid) writer_isbn.append(ISBN) time.sleep(round(np.random.uniform(0.5, 1.4), 2)) if i == bidSearchEnd: print('bid 번호에 대한 도서 ' + str(searchCount) + '권 크롤링이 완료되었습니다.') driver.quit() break book_list = [] book_DF = '' for title, writer, translator, painter, publisher, publishDate, intro, content, authorIntro, categoryTop, categoryMiddle, categoryBottom, bid, ISBN, grade, review, image in zip( title_list, writer_list, translator_list, painter_list, publisher_list, publish_date_list, intro_list, content_list, author_intro_list, category_top_list, category_middle_list, category_bottom_list, bid_list, ISBN_list, grade_list, review_list, image_list): book = { "title": title, "writer": writer, "translator": translator, "painter": painter, "publisher": publisher, "publishDate": publishDate, "intro": intro, "content": content, "authorIntro": authorIntro, "categoryTop": categoryTop, "categoryMiddle": categoryMiddle, "categoryBottom": categoryBottom, "bid": bid, "ISBN": ISBN, "grade": grade, "review": review, "image": image } book_list.append(book) book_DF = pd.DataFrame(book_list) writer_info_list = [] writer_DF = '' for num, name, bookTitle, bid, isbn, link in zip( writer_num, writer_name, writer_book_title, writer_bid, writer_isbn, writer_link): writer_info = { "num": num, "name": name, "bookTitle": bookTitle, "bid": bid, "isbn": isbn, "link": link } writer_info_list.append(writer_info) writer_DF = pd.DataFrame(writer_info_list) end_now = datetime.datetime.now() end_time = end_now.strftime('%Y-%m-%d %H:%M:%S') end_list.append(end_time) complete = 'complete' complete_list.append(complete) errorDetail = '' errorDetail_list.append(errorDetail) crawlerNum_list.append(11) task_list = [] task_DF = '' for taskId, taskContent, str_time, end_time, complete, errorDetail, crawlerNum in zip( taskId_list, taskContent_list, str_list, end_list, complete_list, errorDetail_list, crawlerNum_list): task = { "taskId": taskId, "taskContent": taskContent, "str_time": str_time, "end_time": end_time, "complete": complete, "errorDetail": errorDetail, "crawlerNum": crawlerNum } task_list.append(task) task_DF = pd.DataFrame(task_list) conn = engine.connect() book_DF.to_sql(name='bid_crawling', con=engine, if_exists='append', index=False) writer_DF.to_sql(name='writer_info', con=engine, if_exists='append', index=False) task_DF.to_sql(name='task', con=engine, if_exists='append', index=False) conn.close() searchMok -= 1 bidSearchStart += 50 taskId += 1 time.sleep(1) return HttpResponseRedirect(reverse_lazy('bidIndex'))
def _setup_common_firefox_based_fetcher(self, ff_profile: FirefoxProfile) -> None: """ Performs the common setup procedures for Firefox based fetchers, including Firefox itself :param ff_profile: Firefox Profile created for the webdriver :type ff_profile: FirefoxProfile """ # Get the executor URL self._selenium_executor_url = self._get_selenium_executor_url( self.container_host, self.container_port ) # Install the extensions if self.remove_gdpr: self._install_xpi_extension( self._gdpr_extension_xpi, self._gdpr_extension_xpi_id, ff_profile.extensionsDir, ) if self.export_har: self._install_xpi_extension( self._har_export_extension_xpi, self._har_export_extension_xpi_id, ff_profile.extensionsDir, ) # Enable the network monitoring tools to record HAR ff_profile.set_preference("devtools.netmonitor.enabled", True) ff_profile.set_preference("devtools.toolbox.selectedTool", "netmonitor") ff_profile.set_preference("devtools.netmonitor.har.compress", False) ff_profile.set_preference( "devtools.netmonitor.har.includeResponseBodies", False ) ff_profile.set_preference("devtools.netmonitor.har.jsonp", False) ff_profile.set_preference("devtools.netmonitor.har.jsonpCallback", False) ff_profile.set_preference("devtools.netmonitor.har.forceExport", False) ff_profile.set_preference( "devtools.netmonitor.har.enableAutoExportToFile", False ) ff_profile.set_preference( "devtools.netmonitor.har.pageLoadedTimeout", "2500" ) # Stop updates ff_profile.set_preference("app.update.enabled", False) # Disable JSON view page ff_profile.set_preference("devtools.jsonview.enabled", False) # Set connections to Tor if we need to use Tor if self.use_proxy_type == "tor": ff_profile.set_preference("network.proxy.type", 1) ff_profile.set_preference("network.proxy.socks_version", 5) ff_profile.set_preference("network.proxy.socks", str(self._proxy_host)) ff_profile.set_preference("network.proxy.socks_port", int(self._proxy_port)) ff_profile.set_preference("network.proxy.socks_remote_dns", True) elif self.use_proxy_type == "http": ff_profile.set_preference("network.proxy.type", 1) ff_profile.set_preference("network.proxy.proxy_over_tls", True) ff_profile.set_preference("network.proxy.share_proxy_settings", False) ff_profile.set_preference("network.proxy.http", str(self._proxy_host)) ff_profile.set_preference("network.proxy.http_port", int(self._proxy_port)) ff_profile.set_preference("network.proxy.ssl", str(self._proxy_host)) ff_profile.set_preference("network.proxy.ssl_port", int(self._proxy_port)) ff_profile.set_preference("network.proxy.ftp", str(self._proxy_host)) ff_profile.set_preference("network.proxy.ftp_port", int(self._proxy_port)) if self.disable_cookies: ff_profile.set_preference("network.cookie.cookieBehavior", 2) # Apply the preferences ff_profile.update_preferences() # Set selenium related options for Firefox Browser self._desired_capabilities = webdriver.DesiredCapabilities.FIREFOX.copy() self._selenium_options = webdriver.FirefoxOptions() self._selenium_options.profile = ff_profile if self.disable_javascript: self._selenium_options.preferences.update( { "javascript.enabled": False, } ) if self.export_har: self._selenium_options.add_argument("--devtools")
class TorBrowser: ''' Refactoring for 2020 12 version of Tor Browser Bundle ''' def __init__(self, browser_path, binary_path=None, profile_path=None, executable_path=None, socks_port=9050, control_port=9051, extensions=None, capabilities=None, headless=False, capture_screen=False, url=None): assert browser_path is not None self.browser_path = browser_path self.binary_path = binary_path self.profile_path = profile_path self.executable_path = executable_path self.socks_port = socks_port self.control_port = control_port self.extensions = extensions self.capabilities = capabilities self.headless = headless self.capture_screen = capture_screen self.url = url self.profile = None self.binary = None # firefox self.options = None self.webdriver = None self.tor_controller = TorController() # Start tor process.. self._initialize() def _initialize(self): self._init_browser() self.profile = FirefoxProfile(self.profile_path) self._init_profile() self._init_extensions() self._init_capabilities() self._init_binary() self._init_options() self._init_webdriver() def _init_browser(self): if self.binary_path is None: # tor-browser_en-US/Browser/firefox self.binary_path = os.path.join(self.browser_path, os.path.join('Browser', 'firefox')) if self.profile_path is None: # tor-browser_en-US/Browser/TorBrowser/Data/Browser/profile.default self.profile_path = os.path.join( self.browser_path, os.path.join( 'Browser', os.path.join( 'TorBrowser', os.path.join( 'Data', os.path.join('Browser', 'profile.default'))))) def _init_profile(self): self.profile.set_preference('browser.cache.disk.enable', False) self.profile.set_preference('browser.cache.memory.enable', False) self.profile.set_preference('browser.cache.offline.enable', False) self.profile.set_preference('browser.startup.page', '0') self.profile.set_preference('browser.startup.homepage', 'about:newtab') self.profile.set_preference('network.http.use-cache', False) self.profile.set_preference('network.proxy.type', 1) self.profile.set_preference('network.proxy.socks', '127.0.0.1') self.profile.set_preference('network.proxy.socks_port', self.socks_port) self.profile.set_preference('extensions.torlauncher.promp_at_startup', 0) self.profile.set_preference('network.http.use-cache', False) self.profile.set_preference('webdriver.load.strategy', 'conservative') self.profile.set_preference('extensions.torlauncher.start_tor', False) self.profile.set_preference( 'extensions.torbutton.versioncheck_enabled', False) self.profile.set_preference('permissions.memory_only', False) # update_preference('webdriver.load.strategy', 'normal') # update_preference('app.update.enabled', False) # update_preference('extensions.torbutton.versioncheck_enabled', False) # update_preference('extensions.torbutton.prompted_language', True) # update_preference('extensions.torbutton.socks_port', self.socks_port) # update_preference('extensions.torlauncher.control_port', self.control_port) # update_preference('extensions.torlauncher.start_tor', True) # update_preference('extensions.torbutton.block_dis', False) # update_preference('extensions.torbutton.custom.socks_host', '127.0.0.1') # update_preference('extensions.torbutton.custom.socks_port', self.socks_port) # update_preference('extensions.torbutton.inserted_button', True) # update_preference('extensions.torbutton.launch_warning', False) # update_preference('privacy.spoof_english', 2) # update_preference('extensions.torbutton.loglevel', 2) # update_preference('extensions.torbutton.logmethod', 0) # update_preference('extensions.torbutton.settings_method', 'custom') # update_preference('extensions.torbutton.use_privoxy', False) # update_preference('extensions.torlauncher.control_port', self.control_port) # update_preference('extensions.torlauncher.loglevel', 2) # update_preference('extensions.torlauncher.logmethod', 0) # update_preference('extensions.torlauncher.prompt_at_startup', False) self.profile.update_preferences() def _init_extensions(self): if self.extensions is not None: for extension in self.extensions: self.profile.add_extension(extension) def _init_capabilities(self): if self.capabilities is None: self.capabilities = DesiredCapabilities.FIREFOX self.capabilities.update({ 'handlesAlerts': True, 'databaseEnabled': True, 'javascriptEnabled': True, 'browserConnectionEnabled': True }) def _init_binary(self): self.binary = FirefoxBinary(firefox_path=self.binary_path) self.binary.add_command_line_options('--class', '"Tor Browser"') def _init_options(self): if self.headless is True: self.options = Options() self.options.headless = self.headless def _init_webdriver(self): self.webdriver = webdriver.Firefox( firefox_profile=self.profile, firefox_binary=self.binary, timeout=60, capabilities=self.capabilities, executable_path=self.executable_path, options=self.options) def connect_url(self, url): self.webdriver.get(url) WebDriverWait(self.webdriver, timeout=30).until( expected_conditions.presence_of_element_located( (By.TAG_NAME, 'body'))) def close(self): try: self.tor_controller.stop() self.webdriver.quit() except CannotSendRequest: logger.error('CannotSendRequest while quitting TorBrowserDriver') self.binary.kill() except Exception as e: logger.error('Exception while quitting TorBrowserDriver', e) def init_canvas_permission(self, url): ''' Create a permission DB and add exception for the canvas image extraction. Otherwise screenshots taken by Selenium will be just blank images due to canvas fingerprinting defense in Tor Browser Bundle. ''' import sqlite3 from tld import get_tld connection = sqlite3.connect permission_db = connection( os.path.join(self.profile_path, 'permissions.sqlite')) cursor = permission_db.cursor() # http://mxr.mozilla.org/mozilla-esr31/source/build/automation.py.in cursor.execute("PRAGMA user_version=3") cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts ( id INTEGER PRIMARY KEY, host TEXT, type TEXT, permission INTEGER, expireType INTEGER, expireTime INTEGER, appId INTEGER, isInBrowserElement INTEGER)""") domain = get_tld(url) logger.debug('Adding canvas/extractData permission for %s' % domain) query = """INSERT INTO 'moz_hosts' VALUES (NULL, '%s', 'canvas/extractData', 1, 0, 0, 0, 0);""" % domain cursor.execute(query) permission_db.commit() cursor.close() def take_screenshot(self, save_path): if save_path is not None: save_path = os.path.join(save_path, 'screenshot.png') else: save_path = 'screenshot.png' self.webdriver.get_screenshot_as_file(save_path) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()
class Layer(plonetesting.Layer): profile = None headless = False _browser = 'firefox' def setUp(self): if 'http_address' not in self: raise KeyError("No base layer has set self['http_address']") browser = os.environ.get('GOCEPT_WEBDRIVER_BROWSER') headless = os.environ.get('GOCEPT_SELENIUM_HEADLESS') if headless is None or headless.lower() not in ['true', 'false']: warnings.warn('GOCEPT_SELENIUM_HEADLESS invalid. \ Possible values are true and false. Got: %s.\ Falling back to default (false).' % os.environ.get('GOCEPT_SELENIUM_HEADLESS')) headless = 'false' if headless.lower() == 'true': self.headless = True if browser is None or browser.lower() not in ['chrome', 'firefox']: warnings.warn('GOCEPT_WEBDRIVER_BROWSER invalid. \ Possible values are firefox and chrome. Got: %s.\ Falling back to firefox.' % os.environ.get('GOCEPT_WEBDRIVER_BROWSER')) browser = 'firefox' if browser.lower() == 'chrome': self._browser = 'chrome' else: self.profile = FirefoxProfile( os.environ.get( 'GOCEPT_WEBDRIVER_FF_PROFILE', os.environ.get('GOCEPT_SELENIUM_FF_PROFILE'))) self.profile.native_events_enabled = True self.profile.update_preferences() self._start_selenium() atexit.register(self._stop_selenium) def tearDown(self): self._stop_selenium() # XXX upstream bug, quit should reset session_id self['seleniumrc'].session_id = None del self['seleniumrc'] def _start_selenium(self): if self._browser == 'firefox': options = selenium.webdriver.FirefoxOptions() if self.headless: options.add_argument('-headless') self['seleniumrc'] = selenium.webdriver.Firefox( firefox_profile=self.profile, options=options) if self._browser == 'chrome': options = selenium.webdriver.ChromeOptions() options.add_argument('--disable-dev-shm-usage') if self.headless: options.add_argument('--headless') else: raise NotImplementedError( 'Chromedriver currently only works headless.') self['seleniumrc'] = selenium.webdriver.Chrome( options=options, service_args=['--log-path=chromedriver.log']) def _stop_selenium(self): # Only stop selenium if it is still active. if (self.get('seleniumrc') is None or self['seleniumrc'].session_id is None): return # Quit also removes the tempdir the ff profile is copied in. self['seleniumrc'].quit()
class ProxiedBrowser(object): def __init__(self, dir, tor_port=9150): self.dir = dir self.tor_port = int(tor_port) def open(self): cap_dir = join(self.dir, "caps") try: os.makedirs(cap_dir) except: pass self.proxy = TCP(cap_dir=cap_dir, tor_port=self.tor_port) self.proxy.start() while not self.proxy.running: logger.info("Waiting for proxy to start...") sleep(1) logger.info("Starting Xvfb virtual display") self.vdisplay = Xvfb(width=1280, height=740) self.vdisplay.start() logger.info("Webdriver starting..") self.binary = FirefoxBinary(firefox_path=join(self.dir, FIREFOX_PATH), log_file=open("firefox.log", "w")) self.binary.add_command_line_options("--verbose") self.profile = FirefoxProfile( profile_directory=join(self.dir, FIREFOX_PROFILE)) self.profile.set_preference("network.proxy.socks_port", self.proxy.port) self.profile.set_preference("extensions.torlauncher.start_tor", False) # We started tor manually earlier. self.profile.set_preference("browser.startup.homepage", "") self.profile.set_preference("browser.startup.page", 0) self.profile.update_preferences() try: self.driver = webdriver.Firefox(firefox_binary=self.binary, firefox_profile=self.profile) sleep(2) # hack: wait until homepage etc have loaded. except Exception as ex: self.proxy.close() raise ex return self def __enter__(self): return self.open() def get(self, url): print("Fetching {url}".format(url=url)) self.proxy.consume_results( ) # clear anything previous, e.g the browsers homepage, whatever update checkers etc. self.driver.get(url) capture_files = self.proxy.consume_results() responses = list() for capture_file in capture_files: responses += extract_from_capturefile(capture_file) os.remove(capture_file) return responses def close(self): logging.info("Closing webdriver") self.driver.quit() logging.info("Closing virtual display") self.vdisplay.stop() logging.info("Closing proxy") self.proxy.close() self.proxy.join() def __exit__(self, type, value, traceback): self.close()