def __proxy_pool(self):
"""
Create Proxy connection pool
:raise ProxyRequestError
:return: urllib3.HTTPConnectionPool
"""
try:
self.__server = self.__cfg.proxy if True is self.__cfg.is_standalone_proxy else self.__get_random_proxy()
if self.__get_proxy_type(self.__server) == 'socks':
disable_warnings(InsecureRequestWarning)
if not hasattr(self, '__pm'):
package_module = importlib.import_module('urllib3.contrib.socks')
self.__pm = getattr(package_module, 'SOCKSProxyManager')
pool = self.__pm(self.__server,
num_pools=self.__cfg.threads,
timeout=Timeout(self.__cfg.timeout,
read=self.__cfg.timeout),
block=True)
else:
pool = ProxyManager(self.__server,
num_pools=self.__cfg.threads,
timeout=Timeout(self.__cfg.timeout, read=self.__cfg.timeout),
block=True)
return pool
except (DependencyWarning, ProxySchemeUnknown, ImportError) as error:
raise ProxyRequestError(error)
def __init__(self,
verify=True,
proxies=None,
timeout=None,
max_pool_connections=MAX_POOL_CONNECTIONS,
socket_options=None,
client_cert=None,
):
self._verify = verify
self._proxy_config = ProxyConfiguration(proxies=proxies)
self._pool_classes_by_scheme = {
'http': botocore.awsrequest.AWSHTTPConnectionPool,
'https': botocore.awsrequest.AWSHTTPSConnectionPool,
}
if timeout is None:
timeout = DEFAULT_TIMEOUT
if not isinstance(timeout, (int, float)):
timeout = Timeout(connect=timeout[0], read=timeout[1])
self._cert_file = None
self._key_file = None
if isinstance(client_cert, str):
self._cert_file = client_cert
elif isinstance(client_cert, tuple):
self._cert_file, self._key_file = client_cert
self._timeout = timeout
self._max_pool_connections = max_pool_connections
self._socket_options = socket_options
if socket_options is None:
self._socket_options = []
self._proxy_managers = {}
self._manager = PoolManager(**self._get_pool_manager_kwargs())
self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme
def export_to_s3(args):
"""S3 export routine"""
# Retrieve s3 access and secret keys
access_key = get_secret(
's3_access_key') if not args.s3accesskey else args.s3accesskey
secret_key = get_secret(
's3_secret_key') if not args.s3secretkey else args.s3secretkey
# Initialize minioClient with an endpoint and access/secret keys.
minioClient = Minio('%s:%s' % (args.s3endpoint, args.s3port),
access_key=access_key,
secret_key=secret_key,
secure=args.secure)
minioClient._http.connection_pool_kw['timeout'] = Timeout(
connect=args.timeout, read=3 * args.timeout)
# Make a bucket with the make_bucket API call.
try:
if not minioClient.bucket_exists(args.s3bucket):
minioClient.make_bucket(args.s3bucket, location=args.s3location)
except MaxRetryError:
logging.error(
'Connection timeout! Removing traces older than %d minutes',
args.agemin)
cleanup(args)
pass
except BucketAlreadyOwnedByYou:
pass
except BucketAlreadyExists:
pass
except ResponseError:
raise
else:
# Upload traces to the server
try:
traces = [f for f in files(args.dir)]
traces.sort(key=lambda f: int(''.join(filter(str.isdigit, f))))
# Upload complete traces, exclude most recent log
for trace in traces[:-1]:
minioClient.fput_object(
args.s3bucket,
'%s.%s.sf' % (os.path.basename(trace), args.nodeip),
trace,
metadata={
'x-amz-meta-nodename': args.nodename,
'x-amz-meta-nodeip': args.nodeip,
'x-amz-meta-podname': args.podname,
'x-amz-meta-podip': args.podip,
'x-amz-meta-podservice': args.podservice,
'x-amz-meta-podns': args.podns,
'x-amz-meta-poduuid': args.poduuid
})
os.remove(trace)
logging.info('Uploaded trace %s', trace)
# Upload partial trace without removing it
#minioClient.fput_object(args.s3bucket, os.path.basename(traces[-1]), traces[-1], metadata={'X-Amz-Meta-Trace': 'partial'})
#logging.info('Uploaded trace %s', traces[-1])
except ResponseError:
logging.exception(
'Caught exception while uploading traces to object store')
def chunk_delete_many(self, chunks, cid=None, **kwargs):
"""
:rtype: `list` of either `urllib3.response.HTTPResponse`
or `urllib3.exceptions.HTTPError`, with an extra "chunk"
attribute.
"""
headers = kwargs['headers'].copy()
if cid is not None:
# This is only to get a nice access log
headers['X-oio-chunk-meta-container-id'] = cid
timeout = kwargs.get('timeout')
if not timeout:
timeout = Timeout(CHUNK_TIMEOUT)
def __delete_chunk(chunk_):
try:
resp = self.http_pool.request("DELETE",
chunk_['url'],
headers=headers,
timeout=timeout)
resp.chunk = chunk_
return resp
except HTTPError as ex:
ex.chunk = chunk_
return ex
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
for chunk in chunks:
pile.spawn(__delete_chunk, chunk)
resps = [resp for resp in pile if resp]
return resps
def download_sig(opts, sig, version=None):
"""Download signature from hostname"""
code = None
downloaded = False
useagent = 'ClamAV/0.101.1 (OS: linux-gnu, ARCH: x86_64, CPU: x86_64)'
manager = PoolManager(headers=make_headers(user_agent=useagent),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
timeout=Timeout(connect=10.0, read=60.0))
if version:
path = '/%s.cvd' % sig
filename = os.path.join(opts.workdir, '%s.cvd' % sig)
else:
path = '/%s.cdiff' % sig
filename = os.path.join(opts.workdir, '%s.cdiff' % sig)
try:
req = manager.request('GET', 'http://%s%s' % (opts.hostname, path))
except BaseException as msg:
error("Request error: %s" % msg)
data = req.data
code = req.status
if req.status == 200:
with open(filename, 'wb') as handle:
handle.write(data)
downloaded = os.path.exists(filename)
return downloaded, code
def __init__(self,
verify=True,
proxies=None,
timeout=None,
max_pool_connections=MAX_POOL_CONNECTIONS,
):
self._verify = verify
self._proxy_config = ProxyConfiguration(proxies=proxies)
self._pool_classes_by_scheme = {
'http': botocore.awsrequest.AWSHTTPConnectionPool,
'https': botocore.awsrequest.AWSHTTPSConnectionPool,
}
if timeout is None:
timeout = DEFAULT_TIMEOUT
if not isinstance(timeout, (int, float)):
timeout = Timeout(connect=timeout[0], read=timeout[1])
self._timeout = timeout
self._max_pool_connections = max_pool_connections
self._proxy_managers = {}
self._manager = PoolManager(
strict=True,
timeout=self._timeout,
maxsize=self._max_pool_connections,
ssl_context=self._get_ssl_context(),
)
self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme
def __init__(self,
url,
proxylist=None,
timeout=Timeout(connect=5, read=20)):
self.url = url
headers = {'user-agent': UserAgent().chrome}
if proxylist is not None:
while True:
proxy_host = proxylist.random()
try:
res = requests.get(url,
headers=headers,
proxies={
"http": proxy_host,
"https": proxy_host
},
timeout=timeout)
except (ProxyError, ConnectTimeout, ReadTimeout):
continue
else:
break
else:
res = requests.get(url, headers=headers, timeout=timeout)
self.status_code = res.status_code
try:
response = res.content
# response_encoding = detect(response)
# if response_encoding["encoding"] != "UTF-8" and response_encoding["confidence"] > 0.7:
# response = response.decode("Windows-1250")
except Exception as e:
print(e)
self.soup = None
else:
self.soup = BeautifulSoup(response, "lxml")
def call_api_function(url, method, body=None, headers={'Content-Type': 'application/json'}):
"""
call an API's function and return response
exception: raise an exception if any error occur
:param url: the url
:param method: POST or GET, DELETE
:param body: the body of the request if any
:return: the data structure or None
"""
try:
with PoolManager(retries=5, timeout=Timeout(total=5.0)) as http:
r = http.urlopen(method, url, headers=headers, body=body)
ret_data = (r.status, r.data)
r.close()
except Exception as exc:
e_message = "error getting response from url: " + url
raise ApiCallException(e_message)
else:
if ret_data:
try:
print(ret_data[1])
structured_res = (ret_data[0], json.loads(ret_data[1].decode("utf-8")))
except ValueError as exc:
e_message = str(ret_data[0]) + " : error parsing response from url: " + url
raise ApiCallException(e_message)
else:
structured_res = None
return structured_res
def _load_internet(self):
"""
Description
-----------
boot sequence 3.
check internet connection
Notes
-----
use urllib3 to set connection timeout manually and reduce check time
Return
------
result 0 success
-1 fail
"""
self.net_connected = False
try:
from urllib3 import PoolManager, Timeout , Retry
http = PoolManager(
timeout=Timeout(connect=1.0, read=2.0), retries=Retry(0, redirect=0)
)
response = http.request("HEAD", "https://status.cloud.google.com/")
if response.status == 200: # if internet ok.
self.log.info("pino_boot_loader.py: internet not connected!")
self.net_connected = True
except Exception as E:
self.log.error("pino_boot_loader.py: _load_internet(), " + repr(E))
return -1
else:
return 0
def __init__(self,
etcd_addrs,
key_to_poll,
etcd_scheme="http",
etcd_key=None,
etcd_cert=None,
etcd_ca=None,
poll_timeout=10,
connect_timeout=5):
super(EtcdWatcher, self).__init__(etcd_addrs,
etcd_scheme=etcd_scheme,
etcd_key=etcd_key,
etcd_cert=etcd_cert,
etcd_ca=etcd_ca)
self.etcd_timeout = Timeout(connect=connect_timeout, read=poll_timeout)
self.key_to_poll = key_to_poll
self.next_etcd_index = None
# Forces a resync after the current poll if set. Safe to set from
# another thread. Automatically reset to False after the resync is
# triggered.
self.resync_after_current_poll = False
# Tells the watcher to stop after this poll. One-way flag.
self._stopped = False
self.dispatcher = PathDispatcher()
def __request_api(self, options={}) -> dict:
"""Sent request to diferent data point
Args:
options (dict, optional): options to sent req. Defaults to {}.
Returns:
dict: response of api
"""
url = options.get('url')
headers = options.get('headers')
method = options.get('method')
# Set default timeout to 5 second and retries to 3 using urllib3 PoolManager
retries = Retry(connect=3, read=3, redirect=3)
timeout = Timeout(connect=5.0, read=5.0)
http = PoolManager(retries=retries, timeout=timeout)
# Sent req to api especific
try:
response = http.request(url=url, headers=headers, method=method)
except Exception as err:
return {'Error': err}
return json.loads(response.data.decode('utf-8'))
def get_connection_pool():
retry_policy = Retry(total=5,
backoff_factor=0.1,
status_forcelist=list(range(405, 501)))
timeout_policy = Timeout(read=10, connect=5)
http = PoolManager(retries=retry_policy, timeout=timeout_policy)
return http
def _get_connection_pool():
retry_policy = Retry(total=20,
status_forcelist=list(range(405, 501)),
connect=10,
read=10,
backoff_factor=0.2)
timeout_policy = Timeout(read=50, connect=20)
return PoolManager(retries=retry_policy, timeout=timeout_policy)
def __init__(self):
user_agent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7"
self.headers = {"User-Agent": user_agent}
self.ip_url = "http://icanhazip.com/"
retries = Retry(connect=5, read=25, redirect=5)
self.agent = PoolManager(
10, retries=retries, timeout=Timeout(total=30.0)
)
def _vote(self):
"""Main election thread routine to reconnect and perform election."""
try:
response = self._etcd_client.read(self._key,
timeout=self._interval)
index = response.etcd_index
except etcd.EtcdKeyNotFound:
LOG.debug("Try to become the master - key not found")
self._become_master()
assert False, "_become_master() should not return."
except etcd.EtcdException as e:
# Something bad and unexpected. Log and reconnect.
self._log_exception("read current master", e)
return
LOG.debug("ID of elected master is : %s", response.value)
if response.value:
# If we happen to be on the same server, check if the master
# process is still alive.
self._check_master_process(response.value)
while not self._stopped:
# We know another instance is the master. Wait until something
# changes, giving long enough that it really should do (i.e. we
# expect this read always to return, never to time out).
try:
response = self._etcd_client.read(self._key,
wait=True,
waitIndex=index + 1,
timeout=Timeout(
connect=self._interval,
read=self._ttl * 2))
index = response.etcd_index
except etcd.EtcdKeyNotFound:
# It should be impossible for somebody to delete the object
# without us getting the delete action, but safer to handle it.
LOG.warning("Implausible vanished key - become master")
self._become_master()
except etcd.EtcdEventIndexCleared:
# etcd only keeps a buffer of 1000 events. If that buffer wraps
# before the master refreshes, we get EtcdEventIndexCleared.
# Simply return, which will retry the read and get the new
# etcd index.
LOG.info("etcd index cleared; aborting poll to re-read key.")
return
except etcd.EtcdException as e:
# Something bad and unexpected. Log and reconnect.
self._log_exception("wait for master change", e)
return
LOG.debug("Election key action: %s; new value %s",
response.action, response.value)
if (response.action in ETCD_DELETE_ACTIONS or
response.value is None):
# Deleted - try and become the master.
LOG.info("Leader etcd key went away, attempting to become "
"the elected master")
self._become_master()
class downloadRequest(object):
user_agent = {
'user-agent':
'congressional-record {} (https://github.com/unitedstates/congressional-record)'
.format(VERSION)
}
its_today = datetime.strftime(datetime.today(), '%Y-%m-%d %H:%M')
timeout = Timeout(connect=2.0, read=10.0)
retry = Retry(total=3, backoff_factor=300)
retry.BACKOFF_MAX = 602
http = PoolManager(timeout=timeout,
retries=retry,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
headers=user_agent)
def __init__(self, url, filename):
self.status = False
try:
logging.info('Sending request on {}'.format(self.its_today))
r = self.http.request('GET', url)
logging.debug('Request headers received with code {}'.format(
r.status))
if r.status == 404:
logging.warning('Received 404, not retrying request.')
self.status = 404
elif r.status == 200 and r.data:
logging.info('Considering download request successful.')
logging.info('Sniff sniff: Does this smell like a ZIP file?')
with BytesIO(r.data) as thepackage:
try:
isazip = ZipFile(thepackage)
self.binary_content = r.data
self.status = True
except BadZipfile:
logging.warning(
'File {} is not a valid ZIP file (BadZipFile)'.
format(url))
self.status = False
else:
logging.warning('Unexpected condition, not continuing:\
{}'.format(r.status))
except urllib3.exceptions.MaxRetryError as ce:
logging.warning('Error: %s - Aborting download' % ce)
if self.status == False:
logging.warning('Failed to download file {}'.format(url))
elif self.status == 404:
logging.info('downloadRequester skipping file that returned 404.')
elif self.binary_content:
with open(filename, 'wb') as outfile:
outfile.write(self.binary_content)
logging.info('Wrote {}'.format(filename))
else:
logging.info(
'No download for {} and terminating with unexpected condition.\n'
.format(url))
def __init__(self):
user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
self.headers = {'User-Agent': user_agent}
self.ip_url = 'http://icanhazip.com/'
self.logger = logging.getLogger('gkp')
retries = Retry(connect=5, read=5, redirect=5)
self.agent = ProxyManager('http://localhost:8118/',
retries=retries,
timeout=Timeout(total=60.0))
def __init__(self, configuration=None):
self._pm = PoolManager(timeout=Timeout(connect=1.0),
retries=Retry(1, redirect=0))
if configuration == None:
self._configuration = Configuration()
else:
self._configuration = configuration
self.refreshNetworkList()
self._configuration.isInSync = True
def api_execute(self, path, method, params=None, timeout=None):
if not path.startswith('/'):
raise ValueError('Path does not start with /')
retry = params.pop('retry', None) if isinstance(params, dict) else None
kwargs = {'fields': params, 'preload_content': False}
if method in [self._MGET, self._MDELETE]:
request_executor = self.http.request
elif method in [self._MPUT, self._MPOST]:
request_executor = self.http.request_encode_body
kwargs['encode_multipart'] = False
else:
raise etcd.EtcdException(
'HTTP method {0} not supported'.format(method))
# Update machines_cache if previous attempt of update has failed
if self._update_machines_cache:
self._load_machines_cache()
elif not self._use_proxies and time.time(
) - self._machines_cache_updated > self._machines_cache_ttl:
self._refresh_machines_cache()
machines_cache = self.machines_cache
etcd_nodes = len(machines_cache)
kwargs.update(self._build_request_parameters(etcd_nodes, timeout))
while True:
try:
response = self._do_http_request(retry, machines_cache,
request_executor, method,
path, **kwargs)
return self._handle_server_response(response)
except etcd.EtcdWatchTimedOut:
raise
except etcd.EtcdConnectionFailed as ex:
try:
if self._load_machines_cache():
machines_cache = self.machines_cache
etcd_nodes = len(machines_cache)
except Exception as e:
logger.debug('Failed to update list of etcd nodes: %r', e)
sleeptime = retry.sleeptime
remaining_time = retry.stoptime - sleeptime - time.time()
nodes, timeout, retries = self._calculate_timeouts(
etcd_nodes, remaining_time)
if nodes == 0:
self._update_machines_cache = True
raise ex
retry.sleep_func(sleeptime)
retry.update_delay()
# We still have some time left. Partially reduce `machines_cache` and retry request
kwargs.update(timeout=Timeout(connect=max(1, timeout / 2),
total=timeout),
retries=retries)
machines_cache = machines_cache[:nodes]
def _build_request_parameters(self, etcd_nodes, timeout=None):
kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
if timeout is not None:
kwargs.update(retries=0, timeout=timeout)
else:
_, per_node_timeout, per_node_retries = self._calculate_timeouts(etcd_nodes)
connect_timeout = max(1, per_node_timeout/2)
kwargs.update(timeout=Timeout(connect=connect_timeout, total=per_node_timeout), retries=per_node_retries)
return kwargs
def _vote(self):
"""
Main election thread routine to reconnect and perform election.
"""
try:
response = self._etcd_client.read(self._key,
timeout=self._interval)
index = response.etcd_index
except etcd.EtcdKeyNotFound:
_log.debug("Try to become the master - key not found")
self._become_master()
assert False, "_become_master() should not return."
except (SocketTimeout,
HTTPError,
HTTPException,
etcd.EtcdException) as e:
# Something bad and unexpected. Log and reconnect.
self._log_exception("read current master", e)
return
_log.debug("ID of elected master is : %s", response.value)
while not self._stopped:
# We know another instance is the master. Wait until something
# changes, giving long enough that it really should do (i.e. we
# expect this read always to return, never to time out).
try:
response = self._etcd_client.read(self._key,
wait=True,
waitIndex=index + 1,
timeout=Timeout(
connect=self._interval,
read=self._ttl * 2))
index = response.etcd_index
except etcd.EtcdKeyNotFound:
# It should be impossible for somebody to delete the object
# without us getting the delete action, but safer to handle it.
_log.warning("Implausible vanished key - become master")
self._become_master()
except (SocketTimeout,
HTTPError,
HTTPException,
etcd.EtcdException) as e:
# Something bad and unexpected. Log and reconnect.
self._log_exception("wait for master change", e)
return
if (response.action in ETCD_DELETE_ACTIONS or
response.value is None):
# Deleted - try and become the master.
_log.info("Leader etcd key went away, attempting to become "
"the elected master")
self._become_master()
def http_request_with_custom_ip(address, ip=None, proxies=None):
parts = urlparse(address)
headers = {"Host": parts[1]}
new_address = urlunparse(
(parts[0], ip or parts[1], parts[2], parts[3], parts[4], parts[4]))
lookup_resp = requests.get(new_address,
allow_redirects=False,
timeout=Timeout(8),
proxies=proxies,
headers=headers,
verify=False)
return lookup_resp
def hsts_check():
"""
HSTS check API
url
:return: json type message
"""
# Get URL
url = request.get_data().decode("UTF-8")
# Get host of URL
url = url.replace("https://", "").replace("http://", "")
host = url[:url.find("/")]
site_data = dict()
ssl_info = ''
site_data['hsts'] = False
# Get certificate data
try:
certificate = ssl.get_server_certificate((host, 443))
x_dot_509 = OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM, certificate)
ssl_info = x_dot_509.get_subject().get_components()
except ssl.SSLError as e:
site_data["sslfail"] = str(e)
except socket.gaierror as e:
site_data["sslfail"] = "인증서를 사용하지 않는 사이트입니다."
# HSTS check
try:
http = PoolManager(timeout=Timeout(read=2.0))
request_of_host = http.request(
"GET",
host,
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0)"
},
timeout=2)
response_of_host = request_of_host.headers
if "strict-transport-security" in response_of_host:
site_data["hsts"] = True
except Exception as e:
print(e)
# HSTS check
if ssl_info:
for ssl_data in ssl_info:
site_data[ssl_data[0].decode("UTF-8")] = ssl_data[1].decode(
"UTF-8")
return jsonify(site_data)
def __init__(self,
name=None,
hosts=None,
cloud_id=None,
username=None,
password=None,
use_ssl=False,
verify_certs=False,
assert_hostname=False,
ca_certs=None,
client_cert=None,
client_key=None,
api_key=None,
token=None,
headers=None):
self.name = name
self.hosts = hosts
self.cloud_id = cloud_id
self.auth = f'{username}:{password}' if username and password else None
self.use_ssl = use_ssl
self.verify_certs = verify_certs
self.assert_hostname = assert_hostname
self.ca_certs = ca_certs
self.client_cert = client_cert
self.client_key = client_key
self.api_key = api_key
self.token = token
self.headers = dict(headers) if headers is not None else None
if token:
token_header = {'Authorization': f'Bearer {token}'}
if headers:
headers.update(token_header)
else:
headers = token_header
self.es = Elasticsearch(
hosts=self.hosts.split(',') if self.hosts else None,
cloud_id=cloud_id,
http_auth=self.auth,
use_ssl=use_ssl,
verify_certs=verify_certs,
ca_certs=ca_certs,
client_cert=client_cert,
client_key=client_key,
ssl_show_warn=False,
timeout=Timeout(connect=None, read=None),
api_key=api_key,
headers=headers,
ssl_assert_hostname=assert_hostname,
)
def request(self, url):
"""
Client request SSL
:param str url: request uri
:return: urllib3.HTTPResponse
"""
if self._HTTP_DBG_LEVEL <= self.__debug.level:
self.__debug.debug_request(self._headers, url, self.__cfg.method)
try:
disable_warnings(InsecureRequestWarning)
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan: # directories requests
response = self.__pool.request(self.__cfg.method,
helper.parse_url(url).path,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False)
self.cookies_middleware(is_accept=self.__cfg.accept_cookies, response=response)
else: # subdomains
response = PoolManager().request(self.__cfg.method, url,
headers=self._headers,
retries=self.__cfg.retries,
assert_same_host=False,
redirect=False,
timeout=Timeout(connect=self.__cfg.timeout, read=self.__cfg.timeout))
return response
except MaxRetryError:
if self.__cfg.DEFAULT_SCAN == self.__cfg.scan:
self.__tpl.warning(key='max_retry_error', url=helper.parse_url(url).path)
except HostChangedError as error:
self.__tpl.warning(key='host_changed_error', details=error)
pass
except ReadTimeoutError:
self.__tpl.warning(key='read_timeout_error', url=url)
pass
except ConnectTimeoutError:
self.__tpl.warning(key='connection_timeout_error', url=url)
pass
except SSLError:
if self.__cfg.DEFAULT_SCAN != self.__cfg.scan:
return self._provide_ssl_auth_required()
def api_execute(self, path, method, params=None, timeout=None):
retry = params.pop('retry', None) if isinstance(params, dict) else None
# Update machines_cache if previous attempt of update has failed
if self._update_machines_cache:
self._load_machines_cache()
elif not self._use_proxies and time.time(
) - self._machines_cache_updated > self._machines_cache_ttl:
self._refresh_machines_cache()
machines_cache = self.machines_cache
etcd_nodes = len(machines_cache)
kwargs = self._prepare_common_parameters(etcd_nodes, timeout)
request_executor = self._prepare_request(kwargs, params, method)
while True:
try:
response = self._do_http_request(retry, machines_cache,
request_executor, method,
path, **kwargs)
return self._handle_server_response(response)
except etcd.EtcdWatchTimedOut:
raise
except etcd.EtcdConnectionFailed as ex:
try:
if self._load_machines_cache():
machines_cache = self.machines_cache
etcd_nodes = len(machines_cache)
except Exception as e:
logger.debug('Failed to update list of etcd nodes: %r', e)
sleeptime = retry.sleeptime
remaining_time = retry.stoptime - sleeptime - time.time()
nodes, timeout, retries = self._calculate_timeouts(
etcd_nodes, remaining_time)
if nodes == 0:
self._update_machines_cache = True
self.set_base_uri(
self._base_uri) # trigger Etcd3 watcher restart
raise ex
retry.sleep_func(sleeptime)
retry.update_delay()
# We still have some time left. Partially reduce `machines_cache` and retry request
kwargs.update(timeout=Timeout(connect=max(1, timeout / 2),
total=timeout),
retries=retries)
machines_cache = machines_cache[:nodes]
def is_valid(self, proxy_host, timeout=Timeout(connect=5, read=10)):
try:
response = requests.get("https://canihazip.com/s",
proxies={
"http": proxy_host,
"https": proxy_host
},
timeout=timeout)
except Exception as e:
raise ProxyError(e)
else:
if response.text != proxy_host.replace("http://",
"").split(":")[0]:
raise ProxyError(
"Proxy check failed: {} not used while requesting".format(
proxy_host))
self.checked.update([proxy_host])
def _get_es_hosts(config, logger):
"""
Return list of dicts (a single dict for now) -
that's what ES is expecting.
"""
try:
URL = config.get('Indexing', 'server')
except NoSectionError:
logger.error("Need an [Indexing] section with host and port defined"
" in {} configuration file", " ".join(config.__files__))
return None
except NoOptionError:
host = config.get('Indexing', 'host')
port = config.get('Indexing', 'port')
else:
host, port = URL.rsplit(':', 1)
timeoutobj = Timeout(total=1200, connect=10, read=_read_timeout)
return [dict(host=host, port=port, timeout=timeoutobj),]
def test_timeouts(self):
# This doesn't cover the read timeout because that is harder to mock without adversely affecting the
# environment we're running in, but if we observe that the `requests` library correctly applies the
# connection timeout we can safely assume that would also apply the read timeout.
client = hca.dss.DSSClient()
client.timeout_policy = Timeout(connect=.123, read=.234)
# Prevent unnecessary retries on socket.connect() that don't contribute to code coverage
client.retry_policy = RetryPolicy(connect=0)
with mock.patch('socket.socket.settimeout') as mock_settimeout:
with mock.patch('socket.socket.connect') as mock_connect:
mock_connect.side_effect = socket.timeout
self.assertRaises(ConnectTimeout, client.get_bundle, uuid=str(uuid.uuid4()), replica='gcp')
settimeout_calls, connect_calls = mock_settimeout.mock_calls, mock_connect.mock_calls
# If a domain name resolves to more than one IP (multiple A records with the same name), urllib3 will
# try each one in turn. That's why we may observe multiple calls to settimeout() and connect(). But they
# should come in pairs. We don't care what connect() was called with, only settimout().
self.assertEqual(len(settimeout_calls), len(connect_calls))
self.assertEqual(settimeout_calls, [mock.call(.123)] * len(settimeout_calls))
def dss_client(deployment: Optional[str] = None) -> DSSClient:
"""
Return a DSS client to DSS production or the specified DSS deployment.
:param deployment: The name of a DSS deployment like `dev`, `integration` or `staging`. If None, the production
deployment (`prod`) will be used.
"""
# Work around https://github.com/HumanCellAtlas/dcp-cli/issues/142
hca_config = HCAConfig()
deployment = deployment + "." if deployment else ""
hca_config[
'DSSClient'].swagger_url = f'https://dss.{deployment}data.humancellatlas.org/v1/swagger.json'
# Clear the cached swagger specs that may come from a different deployment. This work-around isn't thread safe but
# neither is the caching iteself.
DSSClient._swagger_spec = None
client = DSSClient(config=hca_config)
client.timeout_policy = Timeout(connect=10, read=40)
return client
