def init_poolmanager(self,
connections,
maxsize,
block=False,
**pool_kwargs):
#pylint: disable=attribute-defined-outside-init
#We need to set self.poolmanager in this fashion
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
assert_hostname=False,
**pool_kwargs)
def init_poolmanager(self,
connections,
maxsize,
block=DEFAULT_POOLBLOCK,
**pool_kwargs):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
assert_hostname=False,
**pool_kwargs)
# Setup the retry strategy
self.max_retries = retries
def init_poolmanager(self,
connections,
maxsize,
block=False,
**pool_kwargs):
"""initialise the PoolManager with the pinned fingerprint"""
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
ca_certs=self._ca_cert_temp.name,
assert_fingerprint=self._cert_fingerprint,
**pool_kwargs)
def init_poolmanager(self,
connections,
maxsize,
block=False,
**pool_kwargs):
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
strict=False,
ssl_version=ssl.PROTOCOL_TLSv1,
**pool_kwargs)
def init_poolmanager(self,
connections,
maxsize,
block=False,
**pool_kwargs):
# Old code had: ssl.PROTOCOL_TLSv1 but there is no upstream to _ssl where the variable actually lives
# Copied the code straight from _ssl for clarity.
# PROTOCOL_SSLv2 = 0
# PROTOCOL_SSLv23 = 2
# PROTOCOL_SSLv3 = 1
# PROTOCOL_TLSv1 = 3
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=3)
def init_poolmanager(self,
connections,
maxsize,
block=False,
**pool_kwargs):
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
ssl_context = create_urllib3_context(ciphers=CIPHERS)
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
ssl_context=ssl_context,
**pool_kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(
num_pools=connections, maxsize=maxsize,
block=block)
test_hosts = [HTTPPoolKey('http', 'localhost', 3000, None, None, None, block, None),
HTTPPoolKey('http', 'localhost.', 3000, None, None, None, block, None)]
for host in test_hosts:
self.poolmanager.pools[host] = \
TestHTTPConnectionPool(host[1], port=host[2])
self.max_retries = TestRetry()
self.max_retries.status_forcelist.add(408)
self.max_retries.status_forcelist.add(500)
self.max_retries.status_forcelist.add(502)
self.max_retries.status_forcelist.add(503)
self.max_retries.status_forcelist.add(504)
self.max_retries.method_whitelist = {'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'PATCH', 'POST'}
def _get_http_session(self):
'''
This is a lazy initializer for the HTTP session.
It does not need to be active until it is required.
'''
if self.http_session is None:
self.http_session = requests.Session()
if not self.http_redirect:
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.poolmanager import PoolManager
import ssl
http_adapter = HTTPAdapter()
http_adapter.poolmanager = PoolManager(
ssl_version=ssl.PROTOCOL_TLSv1)
self.http_session.mount('https://', http_adapter)
return self.http_session
def init_poolmanager(self,
connections,
maxsize,
block=_default_block):
pool_kw = dict()
if block is _default_block:
try:
from requests.adapters import DEFAULT_POOLBLOCK # 1.2.1+
except ImportError:
pass
else:
pool_kw['block'] = DEFAULT_POOLBLOCK
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
ssl_version=ssl.PROTOCOL_TLSv1,
**pool_kw)
def process_size(path, id_list, item_type, asset_type, overwrite):
results = []
summation = 0
path = args.size
spc = psutil.disk_usage(path).free
remain = float(spc) / 1073741824
# now start downloading each file
for item_id in id_list:
url = ASSET_URL.format(item_type, item_id)
logging.info('Request: {}'.format(url))
result = SESSION.get(url)
check_status(result)
try:
if result.json()[asset_type]['status'] == 'active':
download_url = result.json()[asset_type]['location']
#print(download_url)
pool = PoolManager()
response = pool.request("GET",
download_url,
preload_content=False)
max_bytes = 100000000000
content_bytes = response.headers.get("Content-Length")
print("Item-ID: " + str(item_id))
#print(int(content_bytes)/1048576,"MB")
summary = float(content_bytes) / 1073741824
summation = summation + summary
print(format(float(summation), '.2f'), "GB", end='\r')
#print ("Total Size in MB",summation)
else:
result = False
except KeyError:
print(
'Could not check activation status - asset type \'{}\' not found for {}'
.format(asset_type, item_id))
result = False
results.append(result)
#print(remain,"MB")
print("Remaining Space in MB", format(float(remain * 1024), '.2f'))
print("Remaining Space in GB", format(float(remain), '.2f'))
print("Total Size in MB", format(float(summation * 1024), '.2f'))
print("Total Size in GB", format(float(summation), '.2f'))
return results
def init_poolmanager(self, connections, maxsize, block=False):
"""Assemble keyword arguments to be passed to `PoolManager`.
Middlewares are called in reverse order, so if multiple middlewares
define conflicting arguments, the higher-priority middleware will take
precedence. Note: Arguments are passed directly to `PoolManager` and
not to the superclass `init_poolmanager` because the superclass method
does not currently accept **kwargs.
"""
kwargs = {}
for middleware in self.middlewares[::-1]:
value = middleware.before_init_poolmanager(connections, maxsize,
block)
kwargs.update(value or {})
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block)
context = {
"scheme": "http",
"host": "localhost",
"port": 3000,
"block": block,
"maxsize": maxsize
}
test_hosts = [_default_key_normalizer(PoolKey, context)]
for host in test_hosts:
self.poolmanager.pools[host] = \
TestHTTPConnectionPool(host[1], port=host[2])
self.max_retries = TestRetry()
self.max_retries.status_forcelist.add(408)
self.max_retries.status_forcelist.add(500)
self.max_retries.status_forcelist.add(502)
self.max_retries.status_forcelist.add(503)
self.max_retries.status_forcelist.add(504)
self.max_retries.method_whitelist = {
'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'PATCH', 'POST'
}
def init_poolmanager(self, connections, maxsize, **kwargs):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
assert_fingerprint=self.fingerprint)
def init_poolmanager(self, connections, maxsize, **kwargs):
"""Init poolmanager to use ssl version v1."""
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
ssl_version=ssl.PROTOCOL_TLSv1,
**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
block=block, ssl_version=latest_available_ssl_version())
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
source_address=("", self._source_port))
def init_poolmanager(self, connections, maxsize, block=requests.adapters.DEFAULT_POOLBLOCK, **pool_kwargs):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
assert_hostname=False, **pool_kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=ssl.PROTOCOL_TLSv1_2)
def init_poolmanager(self, connections, maxsize, block=False):
"""Set up a poolmanager to use TLS and our cipher list."""
self.poolmanager = PoolManager(
num_pools=connections, maxsize=maxsize, block=block,
ssl_version=ssl.PROTOCOL_TLSv1) # pylint: disable=no-member
pyopenssl.DEFAULT_SSL_CIPHER_LIST = CIPHER_LIST
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
assert_hostname=False,
ssl_version=ssl.PROTOCOL_SSLv23)
def init_poolmanager(self, connections, maxsize, block=False,*args, **kwargs):
context = create_urllib3_context(ciphers=CIPHERS)
kwargs['ssl_context'] = context
self.poolmanager = PoolManager(
num_pools=connections, maxsize=maxsize,
block=block, ssl_version=ssl.PROTOCOL_SSLv3,*args, **kwargs)
def init_poolmanager(self, connections, maxsize, **kwargs):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
ssl_version=SSL_VERSION,
**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
assert_hostname='localhost')
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block)
def download_result_file(url, result_file_directory, result_file_name,
decompress, overwrite):
""" Download file with specified URL and download parameters.
:param result_file_directory: The download result local directory name.
:type result_file_directory: str
:param result_file_name: The download result local file name.
:type result_file_name: str
:param decompress: Determines whether to decompress the ZIP file.
If set to true, the file will be decompressed after download.
The default value is false, in which case the downloaded file is not decompressed.
:type decompress: bool
:param overwrite: Indicates whether the result file should overwrite the existing file if any.
:type overwrite: bool
:return: The download file path.
:rtype: str
"""
if result_file_directory is None:
raise ValueError('result_file_directory cannot be None.')
if result_file_name is None:
result_file_name = "default_file_name"
if decompress:
name, ext = os.path.splitext(result_file_name)
if ext == '.zip':
raise ValueError(
"Result file can't be decompressed into a file with extension 'zip'."
" Please change the extension of the result_file_name or pass decompress=false"
)
zip_file_path = os.path.join(result_file_directory, name + '.zip')
result_file_path = os.path.join(result_file_directory,
result_file_name)
else:
result_file_path = os.path.join(result_file_directory,
result_file_name)
zip_file_path = result_file_path
if os.path.exists(result_file_path) and overwrite is False:
if six.PY3:
raise FileExistsError(
'Result file: {0} exists'.format(result_file_path))
else:
raise OSError('Result file: {0} exists'.format(result_file_path))
pool_manager = PoolManager(ssl_version=ssl.PROTOCOL_SSLv3, )
http_adapter = HTTPAdapter()
http_adapter.poolmanager = pool_manager
s = requests.Session()
s.mount('https://', http_adapter)
r = s.get(url, stream=True, verify=True)
r.raise_for_status()
try:
with open(zip_file_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=4096):
if chunk:
f.write(chunk)
f.flush()
if decompress:
with contextlib.closing(
zipfile.ZipFile(zip_file_path)) as compressed:
first = compressed.namelist()[0]
with open(result_file_path, 'wb') as f:
f.write(compressed.read(first))
except Exception as ex:
raise ex
finally:
if decompress and os.path.exists(zip_file_path):
os.remove(zip_file_path)
return result_file_path
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
assert_hostname=False, **pool_kwargs)
def init_poolmanager(self, *pool_args, **pool_kwargs):
self.poolmanager = PoolManager(
*pool_args,
ssl_version=ssl.PROTOCOL_TLSv1_2,
**pool_kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=self.ssl_protocol)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
cert_file=self.cert_file,
key_file=self.key_file)
