def generator_header(): user_agent = read_random_lines(USER_AGENT_DIR, 5)[0] return {'User-Agent': user_agent, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-us,en;q=0.7,zh-cn;q=0.3', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Connection': 'keep-alive'}
def generator_header(): user_agent = read_random_lines(USER_AGENT_DIR, 5)[0] return [('User-Agent', user_agent), ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), ('Accept-Language', 'en-us,en;q=0.7,zh-cn;q=0.3'), ('Accept-Encoding', 'gzip,deflate'), ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'), ('Connection', 'keep-alive')]
def _init_instance(self, firefox_profile=None, firefox_binary=None, string_proxy=None, timeout=180, capabilities=None, proxy=None, profile_preferences={}, **kwargs): if firefox_profile is None: firefox_profile = LFirefoxProfile(profile_directory=kwargs.get('profile_directory', None)) #, is_temp=kwargs.get('is_temp', False)) firefox_profile.set_preference('browser.cache.disk.capacity', 131072) firefox_profile.set_preference('browser.cache.disk.smart_size.enabled', False) firefox_profile.set_preference('extensions.killspinners.timeout', self.timeout - 2) firefox_profile.set_preference('extensions.killspinners.disablenotify', True) firefox_profile.set_preference('extensions.firebug.showFirstRunPage', False) firefox_profile.set_preference('datareporting.healthreport.uploadEnabled', False) firefox_profile.set_preference('datareporting.healthreport.service.firstRun', False) firefox_profile.set_preference('webdriver.firefox.profile', 'D:\\profiles\\xx') firefox_profile.set_preference('network.proxy.type', 0) if string_proxy: urlinfo = urlparse.urlparse(string_proxy) if urlinfo.scheme == 'ssh': # forwarding_ip = socket.gethostbyname(socket.gethostname()) # forwarding_port = free_port() self.bitvise = Bitvise(urlinfo.hostname, urlinfo.port, username=urlinfo.username, password=urlinfo.password) #, forwarding_ip=forwarding_ip, forwarding_port=forwarding_port) forwarding_ip, forwarding_port = self.bitvise.start() time.sleep(2) # if self.tunnel.login(urlinfo.hostname, urlinfo.username, urlinfo.password, port=urlinfo.port, proxyport=localprot): firefox_profile.set_preference('network.proxy.type', 1) firefox_profile.set_preference('network.proxy.socks', forwarding_ip) firefox_profile.set_preference('network.proxy.socks_port', forwarding_port) firefox_profile.set_preference('network.proxy.socks_remote_dns', True) elif urlinfo.scheme == 'socks5': firefox_profile.set_preference('network.proxy.type', 1) firefox_profile.set_preference('network.proxy.socks', urlinfo.hostname) firefox_profile.set_preference('network.proxy.socks_port', urlinfo.port) firefox_profile.set_preference('network.proxy.socks_remote_dns', True) if kwargs.get('random_ua', False): user_agent = read_random_lines(USER_AGENT_DIR, 5)[0] firefox_profile.set_preference('general.useragent.override', user_agent) for k, v in profile_preferences.items(): firefox_profile.set_preference(k, v) # if sys.platform == 'win32': # executable_path = os.path.join(conf.GECKODRIVER_HOME, 'geckodriver.exe') # else: # executable_path = os.path.join(conf.GECKODRIVER_HOME, 'geckodriver') webdriver.Firefox.__init__(self, firefox_profile=firefox_profile, firefox_binary=firefox_binary, timeout=timeout, capabilities=capabilities, proxy=None) #, executable_path=executable_path) self.set_page_load_timeout(self.timeout) self.implicitly_wait(self.wait_timeout) self.set_script_timeout(self.script_timeout)
def __init__(self, executable_path=PHANTOMJS_PATH, port=0, desired_capabilities=DesiredCapabilities.PHANTOMJS, service_args=[], service_log_path=None, string_proxy=None, timeout=180, **kwargs): self.timeout = timeout self.wait_time = 0.5 user_agent = read_random_lines(USER_AGENT_DIR, 5)[0] _desired_capabilities = { 'phantomjs.page.settings.loadImages': False, 'phantomjs.page.settings.resourceTimeout': '%s' % self.timeout * 1000, 'phantomjs.page.settings.userAgent': kwargs.get('user_agent', user_agent), 'page.settings.loadImages': False, 'page.settings.resourceTimeout': '%s' % self.timeout * 1000, 'page.settings.userAgent': kwargs.get('user_agent', user_agent) } desired_capabilities.update(_desired_capabilities) if string_proxy: urlinfo = urlparse(string_proxy) if urlinfo.scheme == 'socks5': service_args = [ '--proxy=%s:%s' % (urlinfo.hostname, urlinfo.port), '--proxy-type=socks5', ] super(BrowserPhantomJS, self).__init__(executable_path=executable_path, port=0, desired_capabilities=desired_capabilities, service_args=service_args, service_log_path=None) self.wait_timeout = kwargs.get('wait_timeout', self.timeout) self.script_timeout = kwargs.get('script_timeout', self.timeout) self.set_page_load_timeout(self.timeout) # self.set_window_size(random.randint(1000, 2200), random.randint(800, 1500)) self.set_window_size(1680, 1050)