def generate_headers():
""" Генерация заголовков для url запроса """
user_agent = UserAgent()
headers = default_headers()
headers['User-Agent'] = user_agent.random
return headers
def get(url: str, ua_string=None):
"""
function to get http response in text.
:return response object
"""
headers = utils.default_headers()
headers['User-Agent'] = ua_string
# data=b'send exactly these bytes.'
hooks={'response': [print_url, record_hook]}
s = Session()
req = Request('GET', url, headers=headers, hooks=hooks)
# req.register_hook('request', hooks)
try:
prepped = s.prepare_request(req)
resp = s.send(prepped,
stream=True,
verify=True,
proxies=None,
cert='cert.pem',
timeout=(3, 20)
)
except Exception as e:
print(e, url)
# print('Perhaps you meant: \r\n{}'.format(''.join(
# (
# get_domain(Web.crawled[-1]),url
# )
# )))
return -1
return resp.text
def post_json(self, method: str, json_data: dict = None):
''' execute an authenticated POST request against a given method, post provided data'''
headers = default_headers()
headers.update({'Authorization': 'bearer {}'.format(self.token)})
response = post(self.root.format(method),
data=json_data,
headers=headers)
json = response.json()
if (self.config.debug):
print(dumps(json, sort_keys=True, indent=4,
separators=(',', ': ')))
if response.status_code == 408:
raise VehicleAsleepException()
if response.status_code == 401:
raise UnauthorizedException(response.reason)
if response.status_code > 299:
raise Exception('Invalid response: {} {}'.format(
response.status_code, response.reason))
if 'response' in json:
if 'result' in json['response']:
print('[*] Result: {}; reason: {}'.format(
json['response']['result'], json['response']['reason']))
return json
def get_json(self, method: str):
''' execute an authenticated GET request against a given method'''
headers = default_headers()
headers.update({'Authorization': 'bearer {}'.format(self.token)})
response = get(self.root.format(method), headers=headers)
if self.config.debug: print(response.status_code, response.reason)
if response.status_code == 408:
raise VehicleAsleepException()
if response.status_code == 401:
raise UnauthorizedException(response.reason)
if response.status_code > 299:
raise UnexpectedResponseException(
'Invalid response: {} {}]\n{}'.format(response.status_code,
response.reason,
response.content))
json = response.json()
if (self.config.debug):
print(dumps(json, sort_keys=True, indent=4,
separators=(',', ': ')))
return json
def __init__(self,
url: Optional[str] = None,
rules: Optional[str] = None) -> None:
self.headers = default_headers()
self.auth = None
self.proxies = {}
self.hooks = default_hooks()
self.params = {}
self.stream = False
self.verify = True
self.cert = None
self.max_redirects = DEFAULT_REDIRECT_LIMIT
self.trust_env = True
self.cookies = cookiejar_from_dict({})
self.adapters = OrderedDict()
middlewares = [ResurfaceHTTPAdapter(url=url, rules=rules)]
adapter = MiddlewareHTTPAdapter(middlewares)
self.mount("https://", adapter)
self.mount("http://", adapter)
def __init__(self,
config=None,
config_file=None,
debug=None,
http_adapter_kwargs=None):
"""Initialize :class:`ArchiveSession <ArchiveSession>` object with config.
:type config: dict
:param config: (optional) A config dict used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type config_file: str
:param config_file: (optional) Path to config file used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type http_adapter_kwargs: dict
:param http_adapter_kwargs: (optional) Keyword arguments used to initialize the
:class:`requests.adapters.HTTPAdapter <HTTPAdapter>`
object.
:returns: :class:`ArchiveSession` object.
"""
super(ArchiveSession, self).__init__()
http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs
debug = False if not debug else True
self.config = get_config(config, config_file)
self.config_file = config_file
self.cookies.update(self.config.get('cookies', {}))
self.secure = self.config.get('general', {}).get('secure', True)
self.host = self.config.get('general', {}).get('host', 'archive.org')
if 'archive.org' not in self.host:
self.host += '.archive.org'
self.protocol = 'https:' if self.secure else 'http:'
user_email = self.config.get('cookies', dict()).get('logged-in-user')
if user_email:
user_email = user_email.split(';')[0]
user_email = unquote(user_email)
self.user_email = user_email
self.access_key = self.config.get('s3', {}).get('access')
self.secret_key = self.config.get('s3', {}).get('secret')
self.http_adapter_kwargs = http_adapter_kwargs
self.headers = default_headers()
self.headers.update({'User-Agent': self._get_user_agent_string()})
self.headers.update({'Connection': 'close'})
self.mount_http_adapter()
logging_config = self.config.get('logging', {})
if logging_config.get('level'):
self.set_file_logger(
logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'))
if debug or (logger.level <= 10):
self.set_file_logger(
logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'),
'urllib3')
def test_init_with_headers():
from bigchaindb_driver.transport import Transport
headers = {'app_id': 'id'}
transport = Transport('node1', 'node2', headers=headers)
expected_headers = default_headers()
expected_headers.update(headers)
assert transport.pool.connections[0].session.headers == expected_headers
assert transport.pool.connections[1].session.headers == expected_headers
def test_init_with_custom_headers(self):
from bigchaindb_driver.connection import Connection
url = 'http://dummy'
custom_headers = {'app_id': 'id_value', 'app_key': 'key_value'}
connection = Connection(node_url=url, headers=custom_headers)
expected_headers = default_headers()
expected_headers.update(custom_headers)
assert connection.session.headers == expected_headers
def get_client():
headers = default_headers()
headers['User-Agent'] = 'Digital Marketplace (marketplace.service.gov.au)'
client = MailChimp(getenv('MAILCHIMP_SECRET_API_KEY'),
getenv('MAILCHIMP_USERNAME'),
timeout=30.0,
request_headers=headers)
return client
def get_client():
headers = default_headers()
headers['User-Agent'] = 'Digital Marketplace (marketplace.service.gov.au)'
client = MailChimp(
getenv('MAILCHIMP_SECRET_API_KEY'),
getenv('MAILCHIMP_USERNAME'),
timeout=30.0,
request_headers=headers
)
return client
def __init__(self):
_PreparedRequest.__init__(self)
self.headers = default_headers()
self.ssl_options = None
self.host = None
self.port = None
self.af = None
self.decompress = None
self.start_line = None
def get(url, **options):
json = options.get('json', False)
proxy = options.get('proxy', u'')
header = options.get('header', None)
timeout = options.get('timeout', 20)
referer = options.get('referer', None)
session = options.get('session', True)
default_header = utils.default_headers()
default_header['User-Agent'] = USER_AGENT
if referer is not None:
default_header['Referer'] = referer
if isinstance(header, structures.CaseInsensitiveDict):
default_header.update(header)
if proxy:
_proxy = {'https': 'http://%s' % proxy, 'http': 'http://%s' % proxy}
proxy = _proxy
_e = None
_gr = None
_json = None
log.debug('url: %s' % url)
try:
if not session:
_gr = requests.request(method='GET',
url=url,
headers=default_header,
proxies=proxy,
timeout=timeout)
else:
_gr = __session.request(method='GET',
url=url,
headers=default_header,
proxies=proxy,
timeout=timeout)
if json:
_json = _gr.json()
except ValueError as ve:
log.error('ValueError while decoding `%s` to json. %s' % (url, ve))
_e = ve
except (requests.ConnectionError, requests.RequestException) as re:
log.error('requests exception: %s' % re)
_e = re
finally:
if _gr is None or _e is not None:
_response = Response(None, None, None, None, None, error=_e)
else:
_response = Response(_gr.text, _json, _gr.cookies, _gr.headers,
_gr.status_code)
return _response
def logout(self):
"""
Unauthenticate from the client and destroy credential cache.
"""
try:
self.session.cookies = cookiejar_from_dict({})
self.session.headers = default_headers()
return True
except Exception: # pylint:disable=broad-except
return False
def test_init_with_headers():
from bigchaindb_driver.transport import Transport
from bigchaindb_driver.utils import _normalize_nodes
headers = {'app_id': 'id'}
nodes = _normalize_nodes('node1', 'node2', headers=headers)
transport = Transport(*nodes)
expected_headers = default_headers()
expected_headers.update(headers)
connections = transport.pool.connections
assert connections[0]["node"].session.headers == expected_headers
assert connections[1]["node"].session.headers == expected_headers
def fetch_raw_data(url: str, encoding: str = 'windows-1251') -> Any:
headers = default_headers()
headers.update({
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'
})
req = get_request(url, headers)
req.encoding = encoding
return req
def test_driver_init(self, nodes, headers, normalized_nodes):
from bigchaindb_driver.driver import BigchainDB
driver = BigchainDB(*nodes, headers=headers)
nodes = normalized_nodes
headers = {} if not headers else headers
assert driver.nodes == normalized_nodes
assert driver.transport.nodes == normalized_nodes
expected_headers = default_headers()
expected_headers.update(headers)
for conn in driver.transport.pool.connections:
conn.session.headers == expected_headers
assert driver.transactions
assert driver.outputs
def __init__(self,
config=None,
config_file=None,
debug=None,
http_adapter_kwargs=None):
"""Initialize :class:`ArchiveSession <ArchiveSession>` object with config.
:type config: dict
:param config: (optional) A config dict used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type config_file: str
:param config_file: (optional) Path to config file used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type http_adapter_kwargs: dict
:param http_adapter_kwargs: (optional) Keyword arguments used to initialize the
:class:`requests.adapters.HTTPAdapter <HTTPAdapter>`
object.
:returns: :class:`ArchiveSession` object.
"""
super(ArchiveSession, self).__init__()
http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs
debug = False if not debug else True
self.config = get_config(config, config_file)
self.cookies.update(self.config.get('cookies', {}))
# Avoid InsecurePlatformWarning errors on older versions of Python.
if sys.version_info < (2, 7, 9):
self.secure = self.config.get('general', {}).get('secure', False)
else:
self.secure = self.config.get('general', {}).get('secure', True)
self.protocol = 'https:' if self.secure else 'http:'
self.access_key = self.config.get('s3', {}).get('access')
self.secret_key = self.config.get('s3', {}).get('secret')
self.http_adapter_kwargs = http_adapter_kwargs
self.headers = default_headers()
self.headers['User-Agent'] = self._get_user_agent_string()
self._mount_http_adapter()
logging_config = self.config.get('logging', {})
if logging_config.get('level'):
self.set_file_logger(logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'))
if debug or (logger.level <= 10):
self.set_file_logger(logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'),
'requests.packages.urllib3')
def __init__(self, client):
"""
Initialize the authentication module.
:param client: The Client object that will interface with the Auth module.
:type client: :class:`Client`
"""
super(Authenticator, self).__init__()
# Setup session and store host
self.requests = requests
self.host = get_url(client.config["protocol"], client.tenant,
client.config["site.domain"])
self.session.headers = default_headers()
def __init__(self, config=None, config_file=None, debug=None, http_adapter_kwargs=None):
"""Initialize :class:`ArchiveSession <ArchiveSession>` object with config.
:type config: dict
:param config: (optional) A config dict used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type config_file: str
:param config_file: (optional) Path to config file used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type http_adapter_kwargs: dict
:param http_adapter_kwargs: (optional) Keyword arguments used to initialize the
:class:`requests.adapters.HTTPAdapter <HTTPAdapter>`
object.
:returns: :class:`ArchiveSession` object.
"""
super(ArchiveSession, self).__init__()
http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs
debug = False if not debug else True
self.config = get_config(config, config_file)
self.cookies.update(self.config.get("cookies", {}))
# Avoid InsecurePlatformWarning errors on older versions of Python.
if sys.version_info < (2, 7, 9):
self.secure = self.config.get("general", {}).get("secure", False)
else:
self.secure = self.config.get("general", {}).get("secure", True)
self.protocol = "https:" if self.secure else "http:"
self.access_key = self.config.get("s3", {}).get("access")
self.secret_key = self.config.get("s3", {}).get("secret")
self.http_adapter_kwargs = http_adapter_kwargs
self.headers = default_headers()
self.headers["User-Agent"] = self._get_user_agent_string()
self._mount_http_adapter()
logging_config = self.config.get("logging", {})
if logging_config.get("level"):
self.set_file_logger(
logging_config.get("level", "NOTSET"), logging_config.get("file", "internetarchive.log")
)
if debug or (logger.level <= 10):
self.set_file_logger(
logging_config.get("level", "NOTSET"),
logging_config.get("file", "internetarchive.log"),
"requests.packages.urllib3",
)
def __init__(self,
config=None,
config_file=None,
debug=None,
http_adapter_kwargs=None):
"""Initialize :class:`ArchiveSession <ArchiveSession>` object with config.
:type config: dict
:param config: (optional) A config dict used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type config_file: str
:param config_file: (optional) Path to config file used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type http_adapter_kwargs: dict
:param http_adapter_kwargs: (optional) Keyword arguments used to initialize the
:class:`requests.adapters.HTTPAdapter <HTTPAdapter>`
object.
:returns: :class:`ArchiveSession` object.
"""
super(ArchiveSession, self).__init__()
http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs
debug = False if not debug else True
self.config = get_config(config, config_file)
self.config_file = config_file
self.cookies.update(self.config.get('cookies', {}))
self.secure = self.config.get('general', {}).get('secure', True)
self.protocol = 'https:' if self.secure else 'http:'
self.access_key = self.config.get('s3', {}).get('access')
self.secret_key = self.config.get('s3', {}).get('secret')
self.http_adapter_kwargs = http_adapter_kwargs
self.headers = default_headers()
self.headers.update({'User-Agent': self._get_user_agent_string()})
self.headers.update({'Connection': 'close'})
self.mount_http_adapter()
logging_config = self.config.get('logging', {})
if logging_config.get('level'):
self.set_file_logger(logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'))
if debug or (logger.level <= 10):
self.set_file_logger(logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'),
'urllib3')
def test_init_with_headers():
from bigchaindb_driver.transport import Transport
from bigchaindb_driver.utils import normalize_nodes
headers = {'app_id': 'id'}
nodes = normalize_nodes('node1',
{'endpoint': 'node2', 'headers': {'custom': 'c'}},
headers=headers)
transport = Transport(*nodes)
expected_headers = default_headers()
expected_headers.update(headers)
connections = transport.connection_pool.connections
assert connections[0].session.headers == expected_headers
assert connections[1].session.headers == {**expected_headers,
'custom': 'c'}
def __init__(self, phone):
"""
:param phone: Viettel phone number
"""
super().__init__()
self.phone = phone
self.user_id = self.phone_number_2_user_id()
headers = default_headers()
headers.update({
'Password': '******',
'Channel': 'APP',
'User-Agent': 'ViettelPay/3.3.1 (iPhone; iOS 13.3; Scale/3.00)',
'User-Id': self.user_id
})
self.headers = headers # bind headers
def __init__(self,
config=None,
config_file=None,
debug=None,
http_adapter_kwargs=None):
"""Initialize :class:`ArchiveSession <ArchiveSession>` object with config.
:type config: dict
:param config: (optional) A config dict used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type config_file: str
:param config_file: (optional) Path to config file used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:type http_adapter_kwargs: dict
:param http_adapter_kwargs: (optional) Keyword arguments used to initialize the
:class:`requests.adapters.HTTPAdapter <HTTPAdapter>`
object.
"""
super(ArchiveSession, self).__init__()
http_adapter_kwargs = {} if not http_adapter_kwargs else http_adapter_kwargs
debug = False if not debug else True
self.config = get_config(config, config_file)
self.cookies.update(self.config.get('cookies', {}))
# Avoid InsecurePlatformWarning errors on older versions of Python.
if sys.version_info < (2, 7, 9):
self.secure = self.config.get('secure', False)
else:
self.secure = self.config.get('secure', True)
self.protocol = 'https:' if self.secure else 'http:'
self.access_key = self.config.get('s3', {}).get('access')
self.secret_key = self.config.get('s3', {}).get('secret')
self.http_adapter_kwargs = http_adapter_kwargs
self.headers = default_headers()
self.headers['User-Agent'] = self._get_user_agent_string()
self._mount_http_adapter()
logging_config = self.config.get('logging', {})
if logging_config.get('level'):
self.set_file_logger(logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'))
if debug:
self.set_file_logger(logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'),
'requests.packages.urllib3')
def get_basic_rate_html(self):
error = None
data = []
try:
headers = utils.default_headers()
headers[
'User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
response = get(self.URL, timeout=self.TIME_OUT, headers=headers)
response.raise_for_status()
except (ConnectTimeout, HTTPError, ReadTimeout, Timeout,
ConnectionError, RequestException) as err:
error = str(err)
print('ERROR: %s' % error)
else:
data = BeautifulSoup(response.text, 'html.parser')
finally:
return (data, error)
def __get_data(self, url: str) -> DataFrame:
headers = default_headers()
soup = BeautifulSoup(get(self.url).content, 'html.parser')
links = [
'https://www.who.int' + link.get('href')
for link in soup.find_all('a', href=True)
if link.get('href').find('.pdf') > -1
]
dates = [
datetime.strptime(
self.get_filename_from_link(link).rsplit('-')[0], '%Y%m%d')
for link in links
]
reportid = [
self.get_filename_from_link(link).rsplit('-')[2] for link in links
]
list_of_tuples = list(zip(reportid, dates, links))
return DataFrame(list_of_tuples, columns=['Report_ID', 'Date', 'Link'])
def __init__(self, user_id: str = None):
"""
:param user_id: Phone number without prefix '0'
"""
super().__init__()
self.user_id = user_id
# update headers
headers = default_headers()
headers.update({
'Password':
'******',
'Channel':
'APP',
'User-Agent':
'ViettelPay/3.3.1 (iPhone; iOS 13.3; Scale/3.00)',
})
if self.user_id:
headers.update({'User-Id': user_id})
self.headers = headers # bind new headers
def _auth_basic(self, email=None, password=None):
"""
Authenticate by sending email and password combo.
:param email: The email associated with the user's account.
:type email: :class:`string`
:param password: The password associated with the user's account.
:type password: :class:`string`
"""
success = False
if not email or not password:
return success
payload = {
"username": email,
"email": email,
"password": password,
"remember": "yes",
}
url = "{}{}".format(self.host, ENDPOINTS["Auth"]["url"])
self.session.headers = default_headers()
resp = self.session.post(url, data=payload)
if not resp.ok:
raise RuntimeError("Failed login attempt to {}.".format(url))
elif KEYWORDS["KEYWORD_MISSING_ACCOUNT"] in resp.text:
raise RuntimeError(
"Failed login attempt to {}. Account not found.".format(url))
elif KEYWORDS["KEYWORD_INCORRECT_PASSWORD"] in resp.text:
raise RuntimeError(
"Failed login attempt to {}. Password does not match.".format(
url))
elif KEYWORDS["KEYWORD_REQUIRES_SSO"] in resp.text:
raise RuntimeError(
"Failed login attempt to {}. Requires SSO authentication.".
format(url))
else:
success = True
return success
def __init__(self,
config: Mapping | None = None,
config_file: str = "",
debug: bool = False,
http_adapter_kwargs: MutableMapping | None = None):
"""Initialize :class:`ArchiveSession <ArchiveSession>` object with config.
:param config: A config dict used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:param config_file: Path to config file used for initializing the
:class:`ArchiveSession <ArchiveSession>` object.
:param http_adapter_kwargs: Keyword arguments used to initialize the
:class:`requests.adapters.HTTPAdapter <HTTPAdapter>`
object.
:returns: :class:`ArchiveSession` object.
"""
super().__init__()
http_adapter_kwargs = http_adapter_kwargs or {}
debug = bool(debug)
self.config = get_config(config, config_file)
self.config_file = config_file
for ck, cv in self.config.get('cookies', {}).items():
raw_cookie = f'{ck}={cv}'
cookie_dict = parse_dict_cookies(raw_cookie)
if not cookie_dict.get(ck):
continue
cookie = create_cookie(ck,
cookie_dict[ck],
domain=cookie_dict.get(
'domain', '.archive.org'),
path=cookie_dict.get('path', '/'))
self.cookies.set_cookie(cookie)
self.secure: bool = self.config.get('general', {}).get('secure', True)
self.host: str = self.config.get('general',
{}).get('host', 'archive.org')
if 'archive.org' not in self.host:
self.host += '.archive.org'
self.protocol = 'https:' if self.secure else 'http:'
user_email = self.config.get('cookies', {}).get('logged-in-user')
if user_email:
user_email = user_email.split(';')[0]
user_email = unquote(user_email)
self.user_email: str = user_email
self.access_key: str = self.config.get('s3', {}).get('access')
self.secret_key: str = self.config.get('s3', {}).get('secret')
self.http_adapter_kwargs: MutableMapping = http_adapter_kwargs or {}
self.headers = default_headers()
self.headers.update({'User-Agent': self._get_user_agent_string()})
self.headers.update({'Connection': 'close'})
self.mount_http_adapter()
logging_config = self.config.get('logging', {})
if logging_config.get('level'):
self.set_file_logger(
logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'))
if debug or (logger.level <= 10):
self.set_file_logger(
logging_config.get('level', 'NOTSET'),
logging_config.get('file', 'internetarchive.log'),
'urllib3')
def test_format_data_with_data_none():
session = Session()
data = format_data_to_log_string_according_to_headers(session, None, default_headers())
assert data is None
def test_format_data_with_header_json_and_data_none():
session = Session()
headers = default_headers()
headers['Content-Type'] = "application/json"
data = format_data_to_log_string_according_to_headers(session, None, headers)
assert data is None
from httmock import all_requests, response
from requests.utils import default_headers
base_url = "https://example.com/v1/"
headers = default_headers()
headers.update({"content-type": "application/json"})
messages = {
1: "Success",
2: "Bot is running",
3: "Bot is stopped",
1001: "Token is missing",
1002: "Token is wrong",
1003: "Missing required parameters",
1004: "User not found",
1005: "Wrong parameter type",
1006: "Date or time already passed",
1007: "Table not found",
1008: "Table column not found",
1100: "Unknown error"
}
def create_response(success, data, message_code):
return {
"success": success,
"data": data,
"message": {
"code": message_code,
"message": messages[message_code]
def get_history(url: str, **_kw) -> Iterator[Hope]:
"""
:param str url: String url
:param dict headers. See requests.structures.CaseInsensitiveDict
:param str user_agent: String user-agent
:param list cookies: array of strings
Format: ['key1=value1', 'key2=value2', ...]
:param list proxy: array of strings
Format: ['http://proxy:123'] (for http and https)
or ['http=http://proxy:123', 'https=http://secured-proxy:321', 'ftp=http://ftp-proxy:332']
:param bool show_ip: boolean (show ip for each host)
:param bool timeout: int (optional). Timeout for each request (1/100 sec)
:param bool show_request_time: Show request time for each request
:param bool no_error_messages: Do not show any errors
:param bool no_statistic: Do not show statistic of end
:param bool count_only: Show requests count and exit
:param bool try_js: Try check js redirects
:param bool last_only: Show only last request url
:param bool disallow_loops: Do not allow loops (url1->url2->url1-> end)
"""
headers = _kw.pop('headers', default_headers())
if not has_store():
build_store(**_kw)
headers['User-Agent'] = store().args.user_agent
_url = normalize_url(url)
timeout = store().args.timeout
kwargs = {
'headers': headers,
'cookies': store().args.cookies,
'timeout': (timeout / 100 if timeout else None),
}
_prev_url = None
while _url is not None:
if _prev_url is None:
_prev_url = _url
original_url = _url
_url, hook = with_hooks(_url)
response, _url, request_time = find_redirect(url=urljoin(
_prev_url, _url),
**kwargs)
item = Hope(
type='header',
url=response.url,
status=response.status_code,
time=request_time,
headers=dict(response.headers),
hook=hook,
original_url=original_url,
)
if store(
).args.try_js and response.status_code < 300: # only success codes
js_location = __safe_js_redirect(response)
if js_location is not None:
_url = urljoin(_prev_url, js_location)
item = Hope(
type='js',
url=response.url,
status=response.status_code,
time=request_time,
headers=dict(response.headers),
hook=hook,
original_url=original_url,
)
_prev_url = response.url
response.close()
yield item
from requests import Session, Request, utils, exceptions
#CERT = utils.DEFAULT_CA_BUNDLE_PATH
CERT = 'cert.pem'
HEADERS = utils.default_headers()
UA_STRING = 'Curl 7/59.1'
class Web:
crawled = set()
tocrawl = dict()
rel = tuple()
seed = tuple()
def print_url(r, *args, **kwargs):
print(r.url)
def record_hook(r, *args, **kwargs):
r.hook_called = True
# catchme
Web.crawled.add(r.url)
return r
def get(url: str, ua_string=None):
"""
function to get http response in text.
:return response object
"""
headers = utils.default_headers()
headers['User-Agent'] = ua_string
# data=b'send exactly these bytes.'
def _set_headers():
headers = requests_utils.default_headers()
headers["User-Agent"] = UserAgent(verify_ssl=False).random
return headers
