def _create_session(token_v2: str = "") -> Session:
"""
Helper method for creating a session object for API requests.
Arguments
---------
token_v2 : str, optional
Token to use for creating User session.
Defaults to empty string.
Returns
-------
Session
initialised Session object.
"""
retry = Retry(
total=5,
backoff_factor=0.3,
status_forcelist=(502, 503),
method_whitelist=(
"POST",
"HEAD",
"TRACE",
"GET",
"PUT",
"OPTIONS",
"DELETE",
),
)
session = Session()
session.mount("https://", HTTPAdapter(max_retries=retry))
session.cookies = cookiejar_from_dict({"token_v2": token_v2})
return session
def __init__(self, dburi, config, email='', pwd='', user_id=None):
match = re.match(DIRECTUS_API_PATTERN, dburi)
if not match:
raise Exception(
f'{dburi}: Invalid Directus API URL given, should be of pattern {DIRECTUS_API_PATTERN}')
self._reset_cache()
self.direct_db = match.group(3) is None
self.url = match.group(2)
if self.direct_db:
self.user_id = user_id
db = self._db()
self._records = db.tables['records']
self._datasets = db.tables['datasets']
self._datasetrefs = db.tables['datasetrefs']
else:
self.session = requests.Session()
retry = Retry(
backoff_factor=0.3,
status_forcelist=(500, 503, 413, 429),
method_whitelist=('POST', 'PATCH', 'OPTIONS',
'PUT', 'GET', 'TRACE', 'HEAD', 'DELETE')
)
self.session.mount(
f'{parse_url(self.url).scheme}://', HTTPAdapter(max_retries=retry))
self._is_first_flush = True
self.staticToken = False
if config and 'directus_auth_token' in config:
self.staticToken = True
self.session.headers.update(
{'Authorization': f'Bearer {config["directus_auth_token"]}'})
self._refresh_token(config['directus_auth_email'] if config and 'directus_auth_email' in config else email,
config['directus_auth_pwd'] if config and 'directus_auth_pwd' in config else pwd)
def update_dataset(self,
dataframe: pd.DataFrame,
dataset_name: str,
slug: str,
settings: dict = None):
body = {
"label": dataset_name,
"rows": dataframe.to_dict(orient="record")
}
settings_list = []
if (settings != None):
for key in settings.keys():
settings[key]["column_name"] = key
settings_list.append(settings[key])
body["settings"] = settings_list
s = requests.Session()
s.keep_alive = False
retries = Retry(total=5,
backoff_factor=1,
status_forcelist=[502, 503, 504])
s.mount('https://', HTTPAdapter(max_retries=retries))
url = self._base_url_askdata + '/smartbot/agents/' + self._agentId + '/datasets/' + slug + '/sdk'
logging.info("AUTH URL {}".format(url))
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer" + " " + self._token
}
print(body)
response = s.put(url=url, json=body, headers=headers)
response.raise_for_status()
def test_redirect_put_file(self):
"""PUT with file object should work with a redirection response"""
with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool:
retry = Retry(total=3, status_forcelist=[418])
# httplib reads in 8k chunks; use a larger content length
content_length = 65535
data = b"A" * content_length
uploaded_file = io.BytesIO(data)
headers = {
"test-name": "test_redirect_put_file",
"Content-Length": str(content_length),
}
url = "/redirect?target=/echo&status=307"
resp = pool.urlopen(
"PUT",
url,
headers=headers,
retries=retry,
body=uploaded_file,
assert_same_host=False,
redirect=True,
)
assert resp.status == 200
assert resp.data == data
def irmc_scci_update(module, update_url):
if not HAS_REQUESTS:
return 90, "Python 'requests' module not found.", "iRMC module requires 'requests' Module"
session = requests.Session()
retries = Retry(total=5, backoff_factor=0.1)
session.mount('http://', HTTPAdapter(max_retries=retries))
session.mount('https://', HTTPAdapter(max_retries=retries))
url = "http://{0}/{1}".format(module.params['irmc_url'], update_url)
msg = "OK"
try:
data = session.post(url,
verify=module.params['validate_certs'],
auth=HTTPBasicAuth(module.params['irmc_username'],
module.params['irmc_password']))
data.connection.close()
status = data.status_code
if status not in (200, 202, 204):
try:
msg = "POST request was not successful ({0}): {1}".format(
url,
data.json()['error']['message'])
except Exception:
msg = "POST request was not successful ({0}).".format(url)
if "Login required to continue." in data.content:
return 1, "Invalid login data.", "Login required to continue."
except Exception as e:
status = 99
data = traceback.format_exc()
msg = "POST request encountered exception ({0}): {1}".format(
url, str(e))
return status, data, msg
def test_history(self):
retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"]))
assert retry.history == tuple()
connection_error = ConnectTimeoutError("conntimeout")
retry = retry.increment("GET", "/test1", None, connection_error)
history = (RequestHistory("GET", "/test1", connection_error, None, None),)
assert retry.history == history
read_error = ReadTimeoutError(None, "/test2", "read timed out")
retry = retry.increment("POST", "/test2", None, read_error)
history = (
RequestHistory("GET", "/test1", connection_error, None, None),
RequestHistory("POST", "/test2", read_error, None, None),
)
assert retry.history == history
response = HTTPResponse(status=500)
retry = retry.increment("GET", "/test3", response, None)
history = (
RequestHistory("GET", "/test1", connection_error, None, None),
RequestHistory("POST", "/test2", read_error, None, None),
RequestHistory("GET", "/test3", None, 500, None),
)
assert retry.history == history
def get_html_session(url: str = None, cookies: dict = None, ua: str = None, return_type: str = None,
encoding: str = None):
configProxy = config.getInstance().proxy()
session = requests.Session()
if isinstance(cookies, dict) and len(cookies):
requests.utils.add_dict_to_cookiejar(session.cookies, cookies)
retries = Retry(total=configProxy.retry, connect=configProxy.retry, backoff_factor=1,
status_forcelist=[429, 500, 502, 503, 504])
session.mount("https://", TimeoutHTTPAdapter(max_retries=retries, timeout=configProxy.timeout))
session.mount("http://", TimeoutHTTPAdapter(max_retries=retries, timeout=configProxy.timeout))
if configProxy.enable:
session.verify = config.getInstance().cacert_file()
session.proxies = configProxy.proxies()
headers = {"User-Agent": ua or G_USER_AGENT}
session.headers = headers
try:
if isinstance(url, str) and len(url):
result = session.get(str(url))
else: # 空url参数直接返回可重用session对象，无需设置return_type
return session
if not result.ok:
return None
if return_type == "object":
return result
elif return_type == "content":
return result.content
elif return_type == "session":
return result, session
else:
result.encoding = encoding or "utf-8"
return result.text
except requests.exceptions.ProxyError:
print("[-]get_html_session() Proxy error! Please check your Proxy")
except Exception as e:
print(f"[-]get_html_session() failed. {e}")
return None
def fire(self,
url,
revenue_override=None,
additional_revenue=None,
event_multiplier=None,
**kwargs):
parsed_url = urlparse(url)
self.__logger.info(f'Firing Pixel at base url of: {parsed_url}')
query_string_params = parse_qsl(parsed_url.query)
if revenue_override is not None:
query_string_params.append(('override', revenue_override))
if additional_revenue is not None:
query_string_params.append(('additional', additional_revenue))
if event_multiplier is not None:
query_string_params.append(
('eventMultiplier', event_multiplier))
new_query = urlencode(query_string_params)
full_url = ParseResult(parsed_url.scheme, parsed_url.netloc,
parsed_url.path, parsed_url.params,
new_query, parsed_url.fragment).geturl()
self.__logger.info(
f'After url building with overrides, requesting: {full_url}')
pool = self.rest_client.pool_manager
result = pool.request(
'GET',
full_url,
retries=Retry(redirect=False),
headers={
'X-Adzerk-Sdk-Version':
f'adzerk-decision-sdk-python:{__version__}'
})
self.__logger.info(
f'Received response from pixel url: {result.status}')
return (result.status, result.getheader('location'))
def __init__(self, config):
self.confirmAll = config['y']
self.config = config
self.sess = requests.Session()
retryStrategy = Retry(total=5,
status_forcelist=[429, 500, 502, 503, 504],
method_whitelist=["HEAD", "GET", "OPTIONS"])
adapter = HTTPAdapter(max_retries=retryStrategy)
self.sess.mount("https://", adapter)
self.sess.mount("http://", adapter)
self.downloadSize = 0
self.laterDownloadSize = 0
self.courseCode = {}
self.baseurl = self.config['canvasURL'] + '/api/v1'
self.downloadDir = self.config['downloadDir']
self.newInfo = []
self.laterFiles = []
self.laterInfo = []
self.skipfiles = []
self.filesLock = threading.Lock()
self.taskQueue = Queue()
self.downloader = MultithreadDownloader(self.sess, MAX_DOWNLOAD_COUNT)
if not os.path.exists(self.downloadDir):
os.mkdir(self.downloadDir)
def __init__(self, cache_path,
bodhi_url=None,
mdapi_url=None,
icons_url=None,
pdc_url=None,
pagure_url=None):
self.cache_path = cache_path
self.dbpath = join(cache_path, 'search')
self.icons_path = join(cache_path, 'icons')
self.default_icon = 'package_128x128.png'
self.bodhi_url = bodhi_url or "https://bodhi.fedoraproject.org"
self.mdapi_url = mdapi_url or "https://mdapi.fedoraproject.org"
self.icons_url = icons_url or "https://dl.fedoraproject.org/pub/alt/screenshots/"
self.pagure_url = pagure_url or "https://src.fedoraproject.org/api/0"
self.icon_cache = {}
pdc_url = pdc_url or "https://pdc.fedoraproject.org/rest_api/v1"
self.pdc = pdc_client.PDCClient(pdc_url, develop=True, page_size=100)
retries = Retry(connect=10, total=15, backoff_factor=1)
self.pdc.session.mount('https://', HTTPAdapter(max_retries=retries))
self.active_fedora_releases = self._get_active_fedora_releases()
self.create_index()
def post_query(query, access_token):
headers = {"Authorization": "bearer " + access_token}
# endpoint
endpoint = 'https://api.github.com/graphql'
session = requests.Session()
retries = Retry(total=5,
backoff_factor=1,
status_forcelist=[500, 502, 503, 504])
session.mount('https://', HTTPAdapter(max_retries=retries))
session.mount('http://', HTTPAdapter(max_retries=retries))
res = session.post(endpoint, json=query, headers=headers)
if res.status_code == 403:
print('ERROR: HTTPS 403')
time.sleep(3600)
res = session.post(endpoint, json=query, headers=headers)
if res.status_code != 200:
raise Exception("Failed. HTTPS ERROR CODE: {}".format(res.status_code))
return res.json()
def initialize(self):
# Load the state in initialize, use it to store data
# that needs to be accessed across actions
self._state = self.load_state()
# get the asset config
config = self.get_config()
self._base_url = config["base_url"]
self._api_key = config["api_key"]
self._org_id = int(config["org_id"])
# Initialize the requests session that will be used for rest requests
self._request_session = requests.Session()
self._request_session.verify = config.get("verify_server_cert", True)
self._request_session.headers.update({"X-DC-DEVKEY": self._api_key})
# Set proxy vars for the session if they are in the environment
self._request_session.proxies = {}
env_vars = config.get('_reserved_environment_variables', {})
if 'HTTP_PROXY' in env_vars:
self._request_session.proxies['http'] = env_vars['HTTP_PROXY'][
'value']
if 'HTTPS_PROXY' in env_vars:
self._request_session.proxies['https'] = env_vars['HTTPS_PROXY'][
'value']
# Use the retry adapter to retry requests based on certain status codes
retry = Retry(total=3,
backoff_factor=1,
status_forcelist=[408, 429, 500, 502, 503, 504])
retry_adapter = HTTPAdapter(max_retries=retry)
self._request_session.mount("http://", retry_adapter)
self._request_session.mount("https://", retry_adapter)
return phantom.APP_SUCCESS
def __init__(self):
"""Initalizes the `EDGARParser()` Object.
Parsing filings, can change depending on the filing you're working with
and whether you're grabbing the raw filing text or the directory of the filings.
Regardless of what you're parsing, the `EDGARParser()` object will handle most of
the finer details for you.
In cases, where the user needs to parse RSS feeds for the company search, then the
parser will grab all the XML content and convert it to a Python dictionary. Additionally,
it will grab all the next pages and parse thoses if specified.
"""
self.entries_namespace = {
'atom': "http://www.w3.org/2005/Atom",
'atom_with_quote':'{http://www.w3.org/2005/Atom}'
}
self.retry_strategy = Retry(
total=3,
backoff_factor=0.2,
status_forcelist=[429, 500, 502, 503, 504]
)
self.adapter = HTTPAdapter(max_retries=self.retry_strategy)
def __init__(self,
processes=10,
retries=1,
infile='data/all_raw_cleaned.csv',
outfile='data/metadata.csv',
in_urls=None):
self.infile, self.outfile = infile, outfile
self.processes, self.retries = processes, retries
# create session and disable built in retries
self.session = Session()
retry = Retry(total=0)
self.session.mount('http://', HTTPAdapter(max_retries=retry))
# create HTML parser
self.parser = etree.HTMLParser()
# read in URLs
if in_urls is None:
self.urls = []
self.read_in()
else:
self.urls = in_urls
def test_listeners(listener_loader):
os.makedirs(get_path_in_data_folder('logs'), exist_ok=True)
s = requests.Session()
retries = Retry(total=3, backoff_factor=0.5)
s.verify = False
s.mount('http://', HTTPAdapter(max_retries=retries))
s.mount('https://', HTTPAdapter(max_retries=retries))
for l in listener_loader.loaded:
print(f"Testing listener '{l.name}'")
assert isinstance(l, Listener) == True
if l.name in ['http', 'https']:
l['BindIP'] = '127.0.0.1'
l['Port'] = str(random.randint(3000, 6000))
if l.name == 'https':
l['Cert'] = CERT_PATH
l['key'] = KEY_PATH
l.start()
r = s.get(f"{l.name}://127.0.0.1:{l['Port']}/")
assert r.status_code == 404
l.stop()
assert l.running == False
def list_datasets(self):
s = requests.Session()
s.keep_alive = False
retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
s.mount('https://', HTTPAdapter(max_retries=retries))
dataset_url = self._base_url_dataset + '/datasets?agentId=' + self._agentId
response = s.get(url=dataset_url, headers=self._headers)
response.raise_for_status()
r = response.json()
r_df = pd.DataFrame(r)
try:
if r_df.empty:
raise Exception('No datasets in the agent {}'.format(self._agentId))
else:
datasets_df = r_df.loc[:, ['id', 'domain', 'type', 'code', 'name', 'slug', 'description', 'createdBy', 'isActive',
'accessType', 'icon', 'version', 'syncCount', 'visible', 'public', 'createdAt']]
except Exception as e:
datasets_df = r_df
logging.info(e)
return datasets_df
def _auth_session(self, username, password):
"""
Creates session to Hetzner account, authenticates with given credentials and
returns the session, if authentication was successful. Otherwise raises error.
"""
api = self.api[self.account]['auth']
endpoint = api.get('endpoint', self.api[self.account]['endpoint'])
session = requests.Session()
session_retries = Retry(total=10, backoff_factor=0.5)
session_adapter = requests.adapters.HTTPAdapter(max_retries=session_retries)
session.mount('https://', session_adapter)
response = session.request('GET', endpoint + api['GET'].get('url', '/'))
dom = Provider._filter_dom(response.text, api['filter'])
data = Provider._extract_hidden_data(dom)
data[api['user']], data[api['pass']] = username, password
response = session.request('POST', endpoint + api['POST']['url'], data=data)
if Provider._filter_dom(response.text, api['filter']):
LOGGER.error('Hetzner => Unable to authenticate session with %s account \'%s\': '
'Invalid credentials',
self.account, username)
raise AssertionError
LOGGER.info('Hetzner => Authenticate session with %s account \'%s\'',
self.account, username)
return session
def start_session(retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504)):
""" Create Session object with user-agent headers, timeout,
and retry backoff."""
headers = {
'user-agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
'AppleWebKit/537.36 (KHTML, like Gecko)'
'Chrome/68.0.3440.106 Safari/537.36')
}
retry = Retry(total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist)
with Session() as session:
adapter = TimeoutHTTPAdapter(max_retries=retry)
session.headers.update(headers)
session.mount('http:', adapter)
session.mount('https://', adapter)
return session
def list_org_issues(self,
org: str,
state: str = None,
labels: str = None) -> Generator[dict, None, None]:
url = f"{self.GITHUB_ROOT}/search/issues?q=org:{quote(org)}+state:{quote(state)}+archived:false"
if labels:
# Note: GitHub query API expects label to be enclosed in quotes:
quotedLabels = '"' + labels + '"'
url += f"+label:{quote(quotedLabels)}"
# GitHub sometimes returns 5xx errors for this request.
# Retry after 500, 502 response up to 4 times.
max_retries = Retry(status=4, status_forcelist=[500, 502])
adapter = requests.adapters.HTTPAdapter(max_retries=max_retries)
self.session.mount(url, adapter)
while url:
response = self.session.get(url)
response.raise_for_status()
if response.status_code >= 400:
logging.error(response.text)
for item in response.json()["items"]:
yield item
url = response.links.get("next", {}).get("url")
def test_too_many_redirects(self):
with PoolManager() as http:
try:
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={
"target":
"%s/redirect?target=%s/" %
(self.base_url, self.base_url)
},
retries=1,
)
self.fail(
"Failed to raise MaxRetryError exception, returned %r" %
r.status)
except MaxRetryError:
pass
try:
r = http.request(
"GET",
"%s/redirect" % self.base_url,
fields={
"target":
"%s/redirect?target=%s/" %
(self.base_url, self.base_url)
},
retries=Retry(total=None, redirect=1),
)
self.fail(
"Failed to raise MaxRetryError exception, returned %r" %
r.status)
except MaxRetryError:
pass
def scrape_image(url: str, pokenumber: str):
"""
Baixa uma única imagem e a salva em uma pasta definida pelo número da Pokédex.
A imagem é salva para a pasta 'BASE_PATH/pokenumber'.
Parâmetros:
url (str): url da imagem a ser baixada.
pokenumber (str): número da Pokédex, pré-formatado (###).
"""
session = requests.Session()
# previne timeouts
retry = Retry(connect=5, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
img_req = session.get(url)
if img_req.status_code == 200:
img = img_req.content
path = os.path.join(BASE_PATH, pokenumber)
os.makedirs(path, exist_ok=True)
filename = os.path.join(path, url.split("/")[-1])
with open(filename, 'wb') as f:
f.write(img)
def _build_session(self, max_retries):
from requests.adapters import HTTPAdapter
if not isinstance(max_retries, int):
raise ValueError(f'int expected, found {type(max_retries)}.')
elif max_retries < 1:
raise ValueError('max_retries should be greater or equal to 1.')
session = requests.Session()
# mount retries adapter
session.mount(
'http://',
HTTPAdapter(max_retries=Retry(total=max_retries,
method_whitelist=frozenset(
['GET', 'POST']))))
# mount cache adapter
session.mount('http://',
CacheControlAdapter(heuristic=ExpiresAfter(hours=1)))
session.headers['User-Agent'] = USER_AGENT
self.session = session
def get_html_by_browser(url: str = None, cookies: dict = None, ua: str = None, return_type: str = None,
encoding: str = None, use_scraper: bool = False):
configProxy = config.getInstance().proxy()
s = create_scraper(browser={'custom': ua or G_USER_AGENT, }) if use_scraper else requests.Session()
if isinstance(cookies, dict) and len(cookies):
requests.utils.add_dict_to_cookiejar(s.cookies, cookies)
retries = Retry(total=configProxy.retry, connect=configProxy.retry, backoff_factor=1,
status_forcelist=[429, 500, 502, 503, 504])
s.mount("https://", TimeoutHTTPAdapter(max_retries=retries, timeout=configProxy.timeout))
s.mount("http://", TimeoutHTTPAdapter(max_retries=retries, timeout=configProxy.timeout))
if configProxy.enable:
s.verify = config.getInstance().cacert_file()
s.proxies = configProxy.proxies()
try:
browser = mechanicalsoup.StatefulBrowser(user_agent=ua or G_USER_AGENT, session=s)
if isinstance(url, str) and len(url):
result = browser.open(url)
else:
return browser
if not result.ok:
return None
if return_type == "object":
return result
elif return_type == "content":
return result.content
elif return_type == "browser":
return result, browser
else:
result.encoding = encoding or "utf-8"
return result.text
except requests.exceptions.ProxyError:
print("[-]get_html_by_browser() Proxy error! Please check your Proxy")
except Exception as e:
print(f'[-]get_html_by_browser() Failed! {e}')
return None
def create_channel(
self,
name,
icon='https://s3.eu-central-1.amazonaws.com/innaas.smartfeed/icons/groupama/icone/channel/icon_channel_dm.png',
visibility='PRIVATE'):
data = {
"name": name,
"icon": icon,
"agentId": self._agentId,
"visibility": visibility
}
s = requests.Session()
s.keep_alive = False
retries = Retry(total=5,
backoff_factor=1,
status_forcelist=[502, 503, 504])
s.mount('https://', HTTPAdapter(max_retries=retries))
authentication_url = self._base_url_ch + '/channels/'
r = s.post(url=authentication_url, headers=self._headers, json=data)
r.raise_for_status()
return r.json()['id']
def test_respect_retry_after_header_sleep(self, retry_after_header,
respect_retry_after_header,
sleep_duration):
retry = Retry(respect_retry_after_header=respect_retry_after_header)
# Date header syntax can specify an absolute date; compare this to the
# time in the parametrized inputs above.
current_time = mock.MagicMock(return_value=time.mktime(
datetime.datetime(year=2019, month=6, day=3, hour=11).timetuple()))
with mock.patch("time.sleep") as sleep_mock, mock.patch(
"time.time", current_time):
# for the default behavior, it must be in RETRY_AFTER_STATUS_CODES
response = HTTPResponse(
status=503, headers={"Retry-After": retry_after_header})
retry.sleep(response)
# The expected behavior is that we'll only sleep if respecting
# this header (since we won't have any backoff sleep attempts)
if respect_retry_after_header and sleep_duration is not None:
sleep_mock.assert_called_with(sleep_duration)
else:
sleep_mock.assert_not_called()
def requests_retry_session(
retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
):
"""Opinionated wrapper that creates a requests session with a
HTTPAdapter that sets up a Retry policy that includes connection
retries.
If you do the more naive retry by simply setting a number. E.g.::
adapter = HTTPAdapter(max_retries=3)
then it will raise immediately on any connection errors.
Retrying on connection errors guards better on unpredictable networks.
From http://docs.python-requests.org/en/master/api/?highlight=retries#requests.adapters.HTTPAdapter
it says: "By default, Requests does not retry failed connections."
The backoff_factor is documented here:
https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
A default of retries=3 and backoff_factor=0.3 means it will sleep like::
[0.3, 0.6, 1.2]
""" # noqa
session = requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def test_history(self) -> None:
retry = Retry(total=10, allowed_methods=frozenset(["GET", "POST"]))
assert retry.history == tuple()
connection_error = ConnectTimeoutError("conntimeout")
retry = retry.increment("GET", "/test1", None, connection_error)
test_history1 = (RequestHistory("GET", "/test1", connection_error, None, None),)
assert retry.history == test_history1
read_error = ReadTimeoutError(DUMMY_POOL, "/test2", "read timed out")
retry = retry.increment("POST", "/test2", None, read_error)
test_history2 = (
RequestHistory("GET", "/test1", connection_error, None, None),
RequestHistory("POST", "/test2", read_error, None, None),
)
assert retry.history == test_history2
response = HTTPResponse(status=500)
retry = retry.increment("GET", "/test3", response, None)
test_history3 = (
RequestHistory("GET", "/test1", connection_error, None, None),
RequestHistory("POST", "/test2", read_error, None, None),
RequestHistory("GET", "/test3", None, 500, None),
)
assert retry.history == test_history3
def get_latest_agents(arch=None):
"""Get latest user agent of modern browser"""
s = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504])
s.mount('http://', HTTPAdapter(max_retries=retries))
s.mount('https://', HTTPAdapter(max_retries=retries))
try:
response = s.get('https://pylab.co/agents')
except requests.exceptions.RequestException:
return
if response.status_code != 200:
return
data = response.json()
if arch:
if arch in data:
return data[arch]
else:
raise KeyError
else:
return data
def naver_session(nid, npw):
encnm, encpw = encrypt_account(nid, npw)
session = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504])
session.mount('https://', HTTPAdapter(max_retries=retries))
request_headers = {'User-agent': 'Mozilla/5.0'}
bvsd_uuid = uuid.uuid4()
encData = '{"a":"%s-4","b":"1.3.4","d":[{"i":"id","b":{"a":["0,%s"]},"d":"%s","e":false,"f":false},{"i":"%s","e":true,"f":false}],"h":"1f","i":{"a":"Mozilla/5.0"}}' % (
bvsd_uuid, nid, nid, npw)
bvsd = '{"uuid":"%s","encData":"%s"}' % (
bvsd_uuid, lzstring.LZString.compressToEncodedURIComponent(encData))
resp = session.post(
'https://nid.naver.com/nidlogin.login',
data={
'svctype': '0',
'enctp': '1',
'encnm': encnm,
'enc_url':
'http0X0.0000000000001P-10220.0000000.000000www.naver.com',
'url': 'www.naver.com',
'smart_level': '1',
'encpw': encpw,
'bvsd': bvsd
},
headers=request_headers)
finalize_url = re.search(r'location\.replace\("([^"]+)"\)',
resp.content.decode("utf-8")).group(1)
session.get(finalize_url)
return session
def request(method,
url,
payload=None,
retries=5,
backoff_factor=0.3,
status_forcelist=(500, 502, 503, 504),
method_whitelist=frozenset(
['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST']),
is_json=True):
payload = payload or {}
retry = Retry(total=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
method_whitelist=method_whitelist)
http = urllib3.PoolManager()
try:
response = http.request(method, url, fields=payload, retries=retry)
except MaxRetryError as re:
log.info(
"HTTP %s requests exceed max retries with url: %s (reason: %r)" %
(method, url, re.reason))
return {}
if is_json:
try:
ret = json.loads(response.data.decode('utf-8'))
except JSONDecodeError as je:
log.info("[%s] Cannot parse the response from url: %s" %
(je.__class__.__name__, url))
return {}
return ret
return response
