def generate_headers(): """ Генерация заголовков для url запроса """ user_agent = UserAgent() headers = default_headers() headers['User-Agent'] = user_agent.random return headers
def get(url: str, ua_string=None): """ function to get http response in text. :return response object """ headers = utils.default_headers() headers['User-Agent'] = ua_string # data=b'send exactly these bytes.' hooks={'response': [print_url, record_hook]} s = Session() req = Request('GET', url, headers=headers, hooks=hooks) # req.register_hook('request', hooks) try: prepped = s.prepare_request(req) resp = s.send(prepped, stream=True, verify=True, proxies=None, cert='cert.pem', timeout=(3, 20) ) except Exception as e: print(e, url) # print('Perhaps you meant: \r\n{}'.format(''.join( # ( # get_domain(Web.crawled[-1]),url # ) # ))) return -1 return resp.text
def post_json(self, method: str, json_data: dict = None): ''' execute an authenticated POST request against a given method, post provided data''' headers = default_headers() headers.update({'Authorization': 'bearer {}'.format(self.token)}) response = post(self.root.format(method), data=json_data, headers=headers) json = response.json() if (self.config.debug): print(dumps(json, sort_keys=True, indent=4, separators=(',', ': '))) if response.status_code == 408: raise VehicleAsleepException() if response.status_code == 401: raise UnauthorizedException(response.reason) if response.status_code > 299: raise Exception('Invalid response: {} {}'.format( response.status_code, response.reason)) if 'response' in json: if 'result' in json['response']: print('[*] Result: {}; reason: {}'.format( json['response']['result'], json['response']['reason'])) return json
def get_json(self, method: str): ''' execute an authenticated GET request against a given method''' headers = default_headers() headers.update({'Authorization': 'bearer {}'.format(self.token)}) response = get(self.root.format(method), headers=headers) if self.config.debug: print(response.status_code, response.reason) if response.status_code == 408: raise VehicleAsleepException() if response.status_code == 401: raise UnauthorizedException(response.reason) if response.status_code > 299: raise UnexpectedResponseException( 'Invalid response: {} {}]\n{}'.format(response.status_code, response.reason, response.content)) json = response.json() if (self.config.debug): print(dumps(json, sort_keys=True, indent=4, separators=(',', ': '))) return json
def __init__(self, url: Optional[str] = None, rules: Optional[str] = None) -> None: self.headers = default_headers() self.auth = None self.proxies = {} self.hooks = default_hooks() self.params = {} self.stream = False self.verify = True self.cert = None self.max_redirects = DEFAULT_REDIRECT_LIMIT self.trust_env = True self.cookies = cookiejar_from_dict({}) self.adapters = OrderedDict() middlewares = [ResurfaceHTTPAdapter(url=url, rules=rules)] adapter = MiddlewareHTTPAdapter(middlewares) self.mount("https://", adapter) self.mount("http://", adapter)
def __init__(self, config=None, config_file=None, debug=None, http_adapter_kwargs=None): """Initialize :class:`ArchiveSession <ArchiveSession>` object with config. :type config: dict :param config: (optional) A config dict used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type config_file: str :param config_file: (optional) Path to config file used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type http_adapter_kwargs: dict :param http_adapter_kwargs: (optional) Keyword arguments used to initialize the :class:`requests.adapters.HTTPAdapter <HTTPAdapter>` object. :returns: :class:`ArchiveSession` object. """ super(ArchiveSession, self).__init__() http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs debug = False if not debug else True self.config = get_config(config, config_file) self.config_file = config_file self.cookies.update(self.config.get('cookies', {})) self.secure = self.config.get('general', {}).get('secure', True) self.host = self.config.get('general', {}).get('host', 'archive.org') if 'archive.org' not in self.host: self.host += '.archive.org' self.protocol = 'https:' if self.secure else 'http:' user_email = self.config.get('cookies', dict()).get('logged-in-user') if user_email: user_email = user_email.split(';')[0] user_email = unquote(user_email) self.user_email = user_email self.access_key = self.config.get('s3', {}).get('access') self.secret_key = self.config.get('s3', {}).get('secret') self.http_adapter_kwargs = http_adapter_kwargs self.headers = default_headers() self.headers.update({'User-Agent': self._get_user_agent_string()}) self.headers.update({'Connection': 'close'}) self.mount_http_adapter() logging_config = self.config.get('logging', {}) if logging_config.get('level'): self.set_file_logger( logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log')) if debug or (logger.level <= 10): self.set_file_logger( logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log'), 'urllib3')
def test_init_with_headers(): from bigchaindb_driver.transport import Transport headers = {'app_id': 'id'} transport = Transport('node1', 'node2', headers=headers) expected_headers = default_headers() expected_headers.update(headers) assert transport.pool.connections[0].session.headers == expected_headers assert transport.pool.connections[1].session.headers == expected_headers
def test_init_with_custom_headers(self): from bigchaindb_driver.connection import Connection url = 'http://dummy' custom_headers = {'app_id': 'id_value', 'app_key': 'key_value'} connection = Connection(node_url=url, headers=custom_headers) expected_headers = default_headers() expected_headers.update(custom_headers) assert connection.session.headers == expected_headers
def get_client(): headers = default_headers() headers['User-Agent'] = 'Digital Marketplace (marketplace.service.gov.au)' client = MailChimp(getenv('MAILCHIMP_SECRET_API_KEY'), getenv('MAILCHIMP_USERNAME'), timeout=30.0, request_headers=headers) return client
def get_client(): headers = default_headers() headers['User-Agent'] = 'Digital Marketplace (marketplace.service.gov.au)' client = MailChimp( getenv('MAILCHIMP_SECRET_API_KEY'), getenv('MAILCHIMP_USERNAME'), timeout=30.0, request_headers=headers ) return client
def __init__(self): _PreparedRequest.__init__(self) self.headers = default_headers() self.ssl_options = None self.host = None self.port = None self.af = None self.decompress = None self.start_line = None
def get(url, **options): json = options.get('json', False) proxy = options.get('proxy', u'') header = options.get('header', None) timeout = options.get('timeout', 20) referer = options.get('referer', None) session = options.get('session', True) default_header = utils.default_headers() default_header['User-Agent'] = USER_AGENT if referer is not None: default_header['Referer'] = referer if isinstance(header, structures.CaseInsensitiveDict): default_header.update(header) if proxy: _proxy = {'https': 'http://%s' % proxy, 'http': 'http://%s' % proxy} proxy = _proxy _e = None _gr = None _json = None log.debug('url: %s' % url) try: if not session: _gr = requests.request(method='GET', url=url, headers=default_header, proxies=proxy, timeout=timeout) else: _gr = __session.request(method='GET', url=url, headers=default_header, proxies=proxy, timeout=timeout) if json: _json = _gr.json() except ValueError as ve: log.error('ValueError while decoding `%s` to json. %s' % (url, ve)) _e = ve except (requests.ConnectionError, requests.RequestException) as re: log.error('requests exception: %s' % re) _e = re finally: if _gr is None or _e is not None: _response = Response(None, None, None, None, None, error=_e) else: _response = Response(_gr.text, _json, _gr.cookies, _gr.headers, _gr.status_code) return _response
def logout(self): """ Unauthenticate from the client and destroy credential cache. """ try: self.session.cookies = cookiejar_from_dict({}) self.session.headers = default_headers() return True except Exception: # pylint:disable=broad-except return False
def test_init_with_headers(): from bigchaindb_driver.transport import Transport from bigchaindb_driver.utils import _normalize_nodes headers = {'app_id': 'id'} nodes = _normalize_nodes('node1', 'node2', headers=headers) transport = Transport(*nodes) expected_headers = default_headers() expected_headers.update(headers) connections = transport.pool.connections assert connections[0]["node"].session.headers == expected_headers assert connections[1]["node"].session.headers == expected_headers
def fetch_raw_data(url: str, encoding: str = 'windows-1251') -> Any: headers = default_headers() headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0' }) req = get_request(url, headers) req.encoding = encoding return req
def test_driver_init(self, nodes, headers, normalized_nodes): from bigchaindb_driver.driver import BigchainDB driver = BigchainDB(*nodes, headers=headers) nodes = normalized_nodes headers = {} if not headers else headers assert driver.nodes == normalized_nodes assert driver.transport.nodes == normalized_nodes expected_headers = default_headers() expected_headers.update(headers) for conn in driver.transport.pool.connections: conn.session.headers == expected_headers assert driver.transactions assert driver.outputs
def __init__(self, config=None, config_file=None, debug=None, http_adapter_kwargs=None): """Initialize :class:`ArchiveSession <ArchiveSession>` object with config. :type config: dict :param config: (optional) A config dict used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type config_file: str :param config_file: (optional) Path to config file used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type http_adapter_kwargs: dict :param http_adapter_kwargs: (optional) Keyword arguments used to initialize the :class:`requests.adapters.HTTPAdapter <HTTPAdapter>` object. :returns: :class:`ArchiveSession` object. """ super(ArchiveSession, self).__init__() http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs debug = False if not debug else True self.config = get_config(config, config_file) self.cookies.update(self.config.get('cookies', {})) # Avoid InsecurePlatformWarning errors on older versions of Python. if sys.version_info < (2, 7, 9): self.secure = self.config.get('general', {}).get('secure', False) else: self.secure = self.config.get('general', {}).get('secure', True) self.protocol = 'https:' if self.secure else 'http:' self.access_key = self.config.get('s3', {}).get('access') self.secret_key = self.config.get('s3', {}).get('secret') self.http_adapter_kwargs = http_adapter_kwargs self.headers = default_headers() self.headers['User-Agent'] = self._get_user_agent_string() self._mount_http_adapter() logging_config = self.config.get('logging', {}) if logging_config.get('level'): self.set_file_logger(logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log')) if debug or (logger.level <= 10): self.set_file_logger(logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log'), 'requests.packages.urllib3')
def __init__(self, client): """ Initialize the authentication module. :param client: The Client object that will interface with the Auth module. :type client: :class:`Client` """ super(Authenticator, self).__init__() # Setup session and store host self.requests = requests self.host = get_url(client.config["protocol"], client.tenant, client.config["site.domain"]) self.session.headers = default_headers()
def __init__(self, config=None, config_file=None, debug=None, http_adapter_kwargs=None): """Initialize :class:`ArchiveSession <ArchiveSession>` object with config. :type config: dict :param config: (optional) A config dict used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type config_file: str :param config_file: (optional) Path to config file used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type http_adapter_kwargs: dict :param http_adapter_kwargs: (optional) Keyword arguments used to initialize the :class:`requests.adapters.HTTPAdapter <HTTPAdapter>` object. :returns: :class:`ArchiveSession` object. """ super(ArchiveSession, self).__init__() http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs debug = False if not debug else True self.config = get_config(config, config_file) self.cookies.update(self.config.get("cookies", {})) # Avoid InsecurePlatformWarning errors on older versions of Python. if sys.version_info < (2, 7, 9): self.secure = self.config.get("general", {}).get("secure", False) else: self.secure = self.config.get("general", {}).get("secure", True) self.protocol = "https:" if self.secure else "http:" self.access_key = self.config.get("s3", {}).get("access") self.secret_key = self.config.get("s3", {}).get("secret") self.http_adapter_kwargs = http_adapter_kwargs self.headers = default_headers() self.headers["User-Agent"] = self._get_user_agent_string() self._mount_http_adapter() logging_config = self.config.get("logging", {}) if logging_config.get("level"): self.set_file_logger( logging_config.get("level", "NOTSET"), logging_config.get("file", "internetarchive.log") ) if debug or (logger.level <= 10): self.set_file_logger( logging_config.get("level", "NOTSET"), logging_config.get("file", "internetarchive.log"), "requests.packages.urllib3", )
def __init__(self, config=None, config_file=None, debug=None, http_adapter_kwargs=None): """Initialize :class:`ArchiveSession <ArchiveSession>` object with config. :type config: dict :param config: (optional) A config dict used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type config_file: str :param config_file: (optional) Path to config file used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type http_adapter_kwargs: dict :param http_adapter_kwargs: (optional) Keyword arguments used to initialize the :class:`requests.adapters.HTTPAdapter <HTTPAdapter>` object. :returns: :class:`ArchiveSession` object. """ super(ArchiveSession, self).__init__() http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs debug = False if not debug else True self.config = get_config(config, config_file) self.config_file = config_file self.cookies.update(self.config.get('cookies', {})) self.secure = self.config.get('general', {}).get('secure', True) self.protocol = 'https:' if self.secure else 'http:' self.access_key = self.config.get('s3', {}).get('access') self.secret_key = self.config.get('s3', {}).get('secret') self.http_adapter_kwargs = http_adapter_kwargs self.headers = default_headers() self.headers.update({'User-Agent': self._get_user_agent_string()}) self.headers.update({'Connection': 'close'}) self.mount_http_adapter() logging_config = self.config.get('logging', {}) if logging_config.get('level'): self.set_file_logger(logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log')) if debug or (logger.level <= 10): self.set_file_logger(logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log'), 'urllib3')
def test_init_with_headers(): from bigchaindb_driver.transport import Transport from bigchaindb_driver.utils import normalize_nodes headers = {'app_id': 'id'} nodes = normalize_nodes('node1', {'endpoint': 'node2', 'headers': {'custom': 'c'}}, headers=headers) transport = Transport(*nodes) expected_headers = default_headers() expected_headers.update(headers) connections = transport.connection_pool.connections assert connections[0].session.headers == expected_headers assert connections[1].session.headers == {**expected_headers, 'custom': 'c'}
def __init__(self, phone): """ :param phone: Viettel phone number """ super().__init__() self.phone = phone self.user_id = self.phone_number_2_user_id() headers = default_headers() headers.update({ 'Password': '******', 'Channel': 'APP', 'User-Agent': 'ViettelPay/3.3.1 (iPhone; iOS 13.3; Scale/3.00)', 'User-Id': self.user_id }) self.headers = headers # bind headers
def __init__(self, config=None, config_file=None, debug=None, http_adapter_kwargs=None): """Initialize :class:`ArchiveSession <ArchiveSession>` object with config. :type config: dict :param config: (optional) A config dict used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type config_file: str :param config_file: (optional) Path to config file used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :type http_adapter_kwargs: dict :param http_adapter_kwargs: (optional) Keyword arguments used to initialize the :class:`requests.adapters.HTTPAdapter <HTTPAdapter>` object. """ super(ArchiveSession, self).__init__() http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs debug = False if not debug else True self.config = get_config(config, config_file) self.cookies.update(self.config.get('cookies', {})) # Avoid InsecurePlatformWarning errors on older versions of Python. if sys.version_info < (2, 7, 9): self.secure = self.config.get('secure', False) else: self.secure = self.config.get('secure', True) self.protocol = 'https:' if self.secure else 'http:' self.access_key = self.config.get('s3', {}).get('access') self.secret_key = self.config.get('s3', {}).get('secret') self.http_adapter_kwargs = http_adapter_kwargs self.headers = default_headers() self.headers['User-Agent'] = self._get_user_agent_string() self._mount_http_adapter() logging_config = self.config.get('logging', {}) if logging_config.get('level'): self.set_file_logger(logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log')) if debug: self.set_file_logger(logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log'), 'requests.packages.urllib3')
def get_basic_rate_html(self): error = None data = [] try: headers = utils.default_headers() headers[ 'User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' response = get(self.URL, timeout=self.TIME_OUT, headers=headers) response.raise_for_status() except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError, RequestException) as err: error = str(err) print('ERROR: %s' % error) else: data = BeautifulSoup(response.text, 'html.parser') finally: return (data, error)
def __get_data(self, url: str) -> DataFrame: headers = default_headers() soup = BeautifulSoup(get(self.url).content, 'html.parser') links = [ 'https://www.who.int' + link.get('href') for link in soup.find_all('a', href=True) if link.get('href').find('.pdf') > -1 ] dates = [ datetime.strptime( self.get_filename_from_link(link).rsplit('-')[0], '%Y%m%d') for link in links ] reportid = [ self.get_filename_from_link(link).rsplit('-')[2] for link in links ] list_of_tuples = list(zip(reportid, dates, links)) return DataFrame(list_of_tuples, columns=['Report_ID', 'Date', 'Link'])
def __init__(self, user_id: str = None): """ :param user_id: Phone number without prefix '0' """ super().__init__() self.user_id = user_id # update headers headers = default_headers() headers.update({ 'Password': '******', 'Channel': 'APP', 'User-Agent': 'ViettelPay/3.3.1 (iPhone; iOS 13.3; Scale/3.00)', }) if self.user_id: headers.update({'User-Id': user_id}) self.headers = headers # bind new headers
def _auth_basic(self, email=None, password=None): """ Authenticate by sending email and password combo. :param email: The email associated with the user's account. :type email: :class:`string` :param password: The password associated with the user's account. :type password: :class:`string` """ success = False if not email or not password: return success payload = { "username": email, "email": email, "password": password, "remember": "yes", } url = "{}{}".format(self.host, ENDPOINTS["Auth"]["url"]) self.session.headers = default_headers() resp = self.session.post(url, data=payload) if not resp.ok: raise RuntimeError("Failed login attempt to {}.".format(url)) elif KEYWORDS["KEYWORD_MISSING_ACCOUNT"] in resp.text: raise RuntimeError( "Failed login attempt to {}. Account not found.".format(url)) elif KEYWORDS["KEYWORD_INCORRECT_PASSWORD"] in resp.text: raise RuntimeError( "Failed login attempt to {}. Password does not match.".format( url)) elif KEYWORDS["KEYWORD_REQUIRES_SSO"] in resp.text: raise RuntimeError( "Failed login attempt to {}. Requires SSO authentication.". format(url)) else: success = True return success
def __init__(self, config: Mapping | None = None, config_file: str = "", debug: bool = False, http_adapter_kwargs: MutableMapping | None = None): """Initialize :class:`ArchiveSession <ArchiveSession>` object with config. :param config: A config dict used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :param config_file: Path to config file used for initializing the :class:`ArchiveSession <ArchiveSession>` object. :param http_adapter_kwargs: Keyword arguments used to initialize the :class:`requests.adapters.HTTPAdapter <HTTPAdapter>` object. :returns: :class:`ArchiveSession` object. """ super().__init__() http_adapter_kwargs = http_adapter_kwargs or {} debug = bool(debug) self.config = get_config(config, config_file) self.config_file = config_file for ck, cv in self.config.get('cookies', {}).items(): raw_cookie = f'{ck}={cv}' cookie_dict = parse_dict_cookies(raw_cookie) if not cookie_dict.get(ck): continue cookie = create_cookie(ck, cookie_dict[ck], domain=cookie_dict.get( 'domain', '.archive.org'), path=cookie_dict.get('path', '/')) self.cookies.set_cookie(cookie) self.secure: bool = self.config.get('general', {}).get('secure', True) self.host: str = self.config.get('general', {}).get('host', 'archive.org') if 'archive.org' not in self.host: self.host += '.archive.org' self.protocol = 'https:' if self.secure else 'http:' user_email = self.config.get('cookies', {}).get('logged-in-user') if user_email: user_email = user_email.split(';')[0] user_email = unquote(user_email) self.user_email: str = user_email self.access_key: str = self.config.get('s3', {}).get('access') self.secret_key: str = self.config.get('s3', {}).get('secret') self.http_adapter_kwargs: MutableMapping = http_adapter_kwargs or {} self.headers = default_headers() self.headers.update({'User-Agent': self._get_user_agent_string()}) self.headers.update({'Connection': 'close'}) self.mount_http_adapter() logging_config = self.config.get('logging', {}) if logging_config.get('level'): self.set_file_logger( logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log')) if debug or (logger.level <= 10): self.set_file_logger( logging_config.get('level', 'NOTSET'), logging_config.get('file', 'internetarchive.log'), 'urllib3')
def test_format_data_with_data_none(): session = Session() data = format_data_to_log_string_according_to_headers(session, None, default_headers()) assert data is None
def test_format_data_with_header_json_and_data_none(): session = Session() headers = default_headers() headers['Content-Type'] = "application/json" data = format_data_to_log_string_according_to_headers(session, None, headers) assert data is None
from httmock import all_requests, response from requests.utils import default_headers base_url = "https://example.com/v1/" headers = default_headers() headers.update({"content-type": "application/json"}) messages = { 1: "Success", 2: "Bot is running", 3: "Bot is stopped", 1001: "Token is missing", 1002: "Token is wrong", 1003: "Missing required parameters", 1004: "User not found", 1005: "Wrong parameter type", 1006: "Date or time already passed", 1007: "Table not found", 1008: "Table column not found", 1100: "Unknown error" } def create_response(success, data, message_code): return { "success": success, "data": data, "message": { "code": message_code, "message": messages[message_code]
def get_history(url: str, **_kw) -> Iterator[Hope]: """ :param str url: String url :param dict headers. See requests.structures.CaseInsensitiveDict :param str user_agent: String user-agent :param list cookies: array of strings Format: ['key1=value1', 'key2=value2', ...] :param list proxy: array of strings Format: ['http://proxy:123'] (for http and https) or ['http=http://proxy:123', 'https=http://secured-proxy:321', 'ftp=http://ftp-proxy:332'] :param bool show_ip: boolean (show ip for each host) :param bool timeout: int (optional). Timeout for each request (1/100 sec) :param bool show_request_time: Show request time for each request :param bool no_error_messages: Do not show any errors :param bool no_statistic: Do not show statistic of end :param bool count_only: Show requests count and exit :param bool try_js: Try check js redirects :param bool last_only: Show only last request url :param bool disallow_loops: Do not allow loops (url1->url2->url1-> end) """ headers = _kw.pop('headers', default_headers()) if not has_store(): build_store(**_kw) headers['User-Agent'] = store().args.user_agent _url = normalize_url(url) timeout = store().args.timeout kwargs = { 'headers': headers, 'cookies': store().args.cookies, 'timeout': (timeout / 100 if timeout else None), } _prev_url = None while _url is not None: if _prev_url is None: _prev_url = _url original_url = _url _url, hook = with_hooks(_url) response, _url, request_time = find_redirect(url=urljoin( _prev_url, _url), **kwargs) item = Hope( type='header', url=response.url, status=response.status_code, time=request_time, headers=dict(response.headers), hook=hook, original_url=original_url, ) if store( ).args.try_js and response.status_code < 300: # only success codes js_location = __safe_js_redirect(response) if js_location is not None: _url = urljoin(_prev_url, js_location) item = Hope( type='js', url=response.url, status=response.status_code, time=request_time, headers=dict(response.headers), hook=hook, original_url=original_url, ) _prev_url = response.url response.close() yield item
from requests import Session, Request, utils, exceptions #CERT = utils.DEFAULT_CA_BUNDLE_PATH CERT = 'cert.pem' HEADERS = utils.default_headers() UA_STRING = 'Curl 7/59.1' class Web: crawled = set() tocrawl = dict() rel = tuple() seed = tuple() def print_url(r, *args, **kwargs): print(r.url) def record_hook(r, *args, **kwargs): r.hook_called = True # catchme Web.crawled.add(r.url) return r def get(url: str, ua_string=None): """ function to get http response in text. :return response object """ headers = utils.default_headers() headers['User-Agent'] = ua_string # data=b'send exactly these bytes.'
def _set_headers(): headers = requests_utils.default_headers() headers["User-Agent"] = UserAgent(verify_ssl=False).random return headers