def get_mp_connection_pool(self) -> urllib3.HTTPSConnectionPool:
model = self.model
settings = model.settings
m_url = settings[settings.P_MP_DOMAIN].strip()
if not m_url.startswith('http'):
m_url = 'http://' + m_url
url = urlparse(m_url)
if url.netloc.lower() == 'api.arsha.io':
updator = CentralMarketARSHATPriceUpdator
else:
updator = CentralMarketPOSTPriceUpdator
if self.mp_conn_pool is None:
self.mp_conn_pool = urllib3.HTTPSConnectionPool(url.netloc,
maxsize=1,
block=True)
self.market_ready(updator(self.mp_conn_pool, url))
else:
if not url == self.mp_conn_pool.host:
self.mp_conn_pool.close()
self.mp_conn_pool = urllib3.HTTPSConnectionPool(url.netloc,
maxsize=1,
block=True)
self.market_ready(updator(self.mp_conn_pool, url))
return self.mp_conn_pool
def pool_for(self, scheme, host, port):
"""
Given a URL (from which a scheme and host can be extracted),
return a connection pool (potentially with TLS state)
which can be used to connect to the URL.
"""
if scheme is None:
if self._keyfile:
return urllib3.HTTPSConnectionPool(host, port,
key_file=self._keyfile,
cert_file=self._certfile,
ca_certs=self._cafile)
else:
return urllib3.HTTPConnectionPool(host, port)
elif scheme == "http":
return urllib3.HTTPConnectionPool(host, port)
elif scheme == "https":
if self._keyfile:
return urllib3.HTTPSConnectionPool(host, port,
key_file=self._keyfile,
cert_file=self._certfile,
ca_certs=self._cafile)
else:
raise ValueError("SSL requested without providing certificate")
exit(1)
elif scheme == "file":
# FIXME what to do here?
raise ValueError("Unsupported scheme "+scheme)
else:
raise ValueError("Unsupported scheme "+scheme)
def test_simple(self):
try:
pool = urllib3.HTTPSConnectionPool(self.host, self.port)
except urllib3.exceptions.SSLError as e:
self.assertTrue('SSL module is not available' in str(e))
finally:
pool.close()
def __init__(self, token: str, secret: str):
"""
See the documentation at https://api.domeneshop.no/docs/ for
help on how to acquire your API credentials.
:param token: The API client token
:type token: str
:param secret: The API client secret
:type secret: str
"""
self._headers = {
"Authorization":
"Basic {}".format(
base64.b64encode("{}:{}".format(token,
secret).encode()).decode()),
"Accept":
"application/json",
"Content-Type":
"application/json",
"User-Agent":
"domeneshop-python/0.4.2",
}
self._http = urllib3.HTTPSConnectionPool("api.domeneshop.no",
443,
maxsize=5,
block=True,
headers=self._headers,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
def __init__(_self, maxsize: int, block: bool,
**connection_pool_kwargs):
connection_pool_kwargs.get("headers", {}).update({
"connection":
"keep-alive",
"user-agent":
"telegram-bot-client: A Telegram Bot API Python client",
})
# from urllib3.connection.default_socket_options
from socket import IPPROTO_TCP, SOL_SOCKET, TCP_NODELAY, SO_KEEPALIVE
connection_pool_kwargs[
"socket_options"] = connection_pool_kwargs.get(
"socket_options", []) + [
(IPPROTO_TCP, TCP_NODELAY, 1),
(SOL_SOCKET, SO_KEEPALIVE, 1),
]
if self.host.startswith("https://"):
_self.pool = urllib3.HTTPSConnectionPool(
host=self.host[8:],
maxsize=maxsize,
block=block,
**connection_pool_kwargs)
elif self.host.startswith("http://"):
_self.pool = urllib3.HTTPConnectionPool(
host=self.host[7:],
maxsize=maxsize,
block=block,
**connection_pool_kwargs)
else:
raise TelegramBotException(
"Telegram Bot API only supports https:// and http://")
def test_simple(self):
pool = urllib3.HTTPSConnectionPool(self.host, self.port)
self.addCleanup(pool.close)
try:
pool.request('GET', '/')
except urllib3.exceptions.SSLError as e:
self.assertIn('SSL module is not available', str(e))
def __init__(self, info: PluginInfo):
self._log = logging.getLogger(info.name)
self._simul_downloads = dynamic_data.USING_CORES
self._info = info # info about the module
self._temp_path = dynamic_data.TEMP_DIR.joinpath(
self.name) # module temp path
self._download_path = dynamic_data.DOWNLOAD_DIR.joinpath(
self.name) # module download path
self._save_state_file = dynamic_data.SAVESTAT_DIR.joinpath(
self.name + '_save.json') # module save path
try:
create_dir_rec(self._temp_path)
create_dir_rec(self._download_path)
except PermissionError:
raise PluginException(
'Can not create default plugin paths, due to a permission error.'
)
self._last_update = datetime(1970, 1, 1)
self._unit = 'item'
self._download_data = {}
self._downloader = urllib3.HTTPSConnectionPool(
self.info.host,
maxsize=self._simul_downloads,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
def post(self,
endpoint,
data,
apiTimeout: int = None,
sslVerification: bool = None):
try:
import urllib3
http = urllib3.HTTPSConnectionPool(f"10.75.20.60",
port=443,
cert_reqs='CERT_NONE',
assert_hostname=False)
r = http.request('POST',
f"/axl/",
headers=self._default_header,
body=data)
t = r.read()
val = self.__central_handler(
url=f"{self.endpoint}{endpoint}",
headers=self._default_header,
data=data,
verify=sslVerification
if sslVerification is not None else self._default_ssl_verify,
timeout=apiTimeout
if apiTimeout is not None else self._default_timeout,
)
return val
except Exception as e:
self._log.write_log(
data=
f"RequestsConnector failed with error {errorStackTrace(e)}",
level=LogFactory.error)
r = Response()
r.status_code = 500
return r
def analyser(json_file):
"""Analyse JSON data from Tor2web node."""
print json_file
pool = urllib3.HTTPSConnectionPool("127.0.0.1",
443,
timeout=10,
cert_reqs='CERT_NONE',
assert_hostname=False)
json_text = read_file(json_file)
json_data = valid_json(json_text)
if not json_data:
print "Error: %s" % json_text
raise SystemExit
dateday = json_data["date"]
if json_data:
for hidden_service in json_data["hidden_services"]:
access_count = hidden_service["access_count"]
onion_id = hidden_service["id"]
onion_url = 'http://' + onion_id + '.onion/'
print onion_url
data = '{"url": "' + onion_url + '"}'
url = 'https://127.0.0.1/address/'
content_type = {'Content-Type': 'application/json'}
pool.urlopen('POST', url, headers=content_type, body=data)
url = url + onion_id + "/popularity/"
data = '{"date": "' + dateday + '", "tor2web_access_count": '
data = data + str(access_count) + '}'
print data
pool.urlopen('PUT', url, headers=content_type, body=data)
def main():
"""Main function."""
# Gather all backlink information from the rest
timestamp = datetime.datetime.now().strftime("%y-%m-%d")
stats_dir = "/popularity_stats/" + timestamp + "/"
url = 'https://127.0.0.1:45454/address/online/'
pool = urllib3.HTTPSConnectionPool("127.0.0.1",
45454,
timeout=10,
cert_reqs='CERT_NONE',
assert_hostname=False)
links = pool.request('GET', url).data
links = links.replace(".onion/", "").replace("http://", "").split('\n')
for onion_id in links:
try:
# Random delay 3min + 1-60 seconds
delay_time = 180 + random.randrange(1, 60)
time.sleep(delay_time)
if not onion_id:
continue
content_type = {'Content-Type': 'application/json'}
onion_url = 'http://' + onion_id + '.onion/'
print onion_url
backlinks = str(get_backlinks(onion_url))
url = 'https://127.0.0.1:45454/address/' + onion_id + "/popularity/"
data = '{"date": "' + timestamp + '", "tor2web_access_count": '
data = data + '0, "backlinks": ' + backlinks + '}'
print data
save_popularity_data(data, onion_id)
pool.urlopen('PUT', url, headers=content_type, body=data)
except Exception:
import traceback
print 'generic exception: ' + traceback.format_exc()
def _get_connection_pool():
if DEBUG:
return urllib3.HTTPConnectionPool('127.0.0.1:8000')
else:
return urllib3.HTTPSConnectionPool('api.nivad.io',
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
def test_simple(self):
import urllib3
pool = urllib3.HTTPSConnectionPool(self.host, self.port)
try:
pool.request('GET', '/')
except urllib3.exceptions.SSLError as e:
self.assertTrue('SSL module is not available' in str(e))
def get_job(ip_address, header, job_id):
""" Get Job details """
url = 'https://{0}/api/JobService/Jobs({1})'.format(ip_address, job_id)
pool = urllib3.HTTPSConnectionPool(ip_address,
port=443,
cert_reqs='CERT_NONE',
assert_hostname=False)
return pool.urlopen('GET', url, headers=header)
def _test():
# Setting retries to 0 so that metrics are recorded only once
pool = urllib3.HTTPSConnectionPool('localhost:%d' % server.port,
retries=0)
try:
pool.request('GET', '/')
except Exception:
pass
def _test():
# Setting retries to 0 so that metrics are recorded only once
pool = urllib3.HTTPSConnectionPool("localhost:%d" % server.port,
retries=0)
try:
pool.urlopen("GET", "/")
except Exception:
pass
def test_simple(self):
with urllib3.HTTPSConnectionPool(
self.host, self.port, cert_reqs="NONE"
) as pool:
try:
pool.request("GET", "/")
except urllib3.exceptions.SSLError as e:
assert "SSL module is not available" in str(e)
def test_simple(self):
try:
pool = urllib3.HTTPSConnectionPool(self.host,
self.port,
cert_reqs="NONE")
except urllib3.exceptions.SSLError as e:
assert "SSL module is not available" in str(e)
finally:
pool.close()
def __init__(self, appliance, username='', password='', token='', cert_validation=False, verbose=True):
"""
Actifio instance:
:appliance: IP or FQDN of the appliance
:username: Username to login to the appliance
:password: Password
"""
# vendor key is fixed
vendorkey = "195B-4370-2506-0A60-1F41-5829-067B-603C-6737-1244-0A11-6742-0745-4023-1A"
# functools need a __name__ for the wraps
self.__name__ = "Actifio"
# compose the URI for the API
self._apiBase = "/actifio/api"
if token == '' and username == '':
raise ActUserError ("Either 'username' or 'token' required for authentication.")
if token != '':
try:
token_json = json.loads (base64.decodebytes(token))
except:
raise ActUserError ("Incorrect token format. Use 'act_gentoken' command to generate a authentication token")
self.username = token_json["username"]
self.password = token_json["password"]
else:
self.username = username
self.password = password
self.appliance = appliance
self.version = 'not_known'
#sessionid is unique per user in an appliance, and this is global
if Actifio._sessionid.get(appliance) == None:
Actifio._sessionid.update({appliance: {}})
if Actifio._sessionid[appliance].get(self.username) == None:
Actifio._sessionid[appliance].update({ self.username: {}})
Actifio._sessionid[appliance][self.username].update({ "sessionid": "" })
# Actifio._sessionid[appliance][self.username].update({ "password": self.password})
# compose the login params
self._loginParams = "/login?name=" + urlencode_str(self.username)
self._loginParams += "&password="******"&vendorkey=" + vendorkey
self._loginURI = self._apiBase + self._loginParams
self._infoURI = self._apiBase + "/info/"
self._taskURI = self._apiBase + "/task/"
self._sargURI = self._apiBase + "/report/"
# create the https poolmanager
cert_str = 'CERT_REQUIRED' if cert_validation else 'CERT_NONE'
timeout = urllib3.util.timeout.Timeout(connect=5.0)
self._httppool = urllib3.HTTPSConnectionPool(host=appliance, port=443, cert_reqs=cert_str, timeout=timeout)
self._verbose = verbose
def _crawl_articles_worker(output_file: str, article_urls: List[str],
queue: Queue):
pool = urllib3.HTTPSConnectionPool('news.naver.com')
with open(output_file, 'w', encoding='utf-8') as fp:
for article_url in article_urls:
fp.write(_get_article_content(pool, article_url) + '\n')
queue.put(True)
queue.put(None)
def test_simple(self):
pool = urllib3.HTTPSConnectionPool(self.host,
self.port,
cert_reqs="NONE")
self.addCleanup(pool.close)
try:
pool.request("GET", "/")
except urllib3.exceptions.SSLError as e:
self.assertIn("SSL module is not available", str(e))
def run_packet_tunnel_tests(self, test_sites, expected_egress_ip_addresses,
user_agent):
import dns.resolver
import urllib3
if len(test_sites) == 0:
output = {
'PT-DNS': 'FAIL: No test sites provided',
'PT-HTTPS': 'FAIL: No test sites provided'
}
else:
# TODO: Perform the test on multiple sites and record each result
url = test_sites[0]['url']
expected_ip = test_sites[0]['expected_ip']
# packet tunnel dns test. Resolve the url through the tunnel
output = {'PT-DNS': 'FAIL', 'PT-HTTPS': 'FAIL'}
fqdn = ''
remote_host = expected_ip
remote_port = 443
split_url = url.split('://')
if len(split_url) >= 2:
parts = split_url[1].split('/', 2)
fqdn = parts[0]
path = '/'
if len(parts) >= 2:
path += parts[1]
resolver = dns.resolver.Resolver(
configure=False) # Don't use the system resolver settings
resolver.nameservers = ['10.0.0.2']
answer = resolver.query(fqdn, source=self.tun_source_ip_address)
for rr in answer.rrset:
if rr.address == expected_ip:
print 'Packet Tunnel DNS Test successful'
output['PT-DNS'] = 'PASS: {0} resolved to {1}'.format(
fqdn, rr.address)
break
pool = urllib3.HTTPSConnectionPool(
host=fqdn,
port=remote_port,
maxsize=2,
source_address=(self.tun_source_ip_address,
self.tun_source_port))
response = pool.request('GET',
path,
headers={"User-Agent": user_agent},
release_conn=True)
egress_ip_address = response.data.strip()
is_proxied = (egress_ip_address in expected_egress_ip_addresses)
if is_proxied:
output['PT-HTTPS'] = 'PASS'
return output
def get(self, host, params=()):
# type: (object, object) -> object
"""Get metadata by url"""
self.__is_server_online(host)
self.__disable_verbose()
self.__parse_params(params)
scheme, host = urlparse(host).scheme, urlparse(host).netloc
self.DEFAULT_HTTP_PROTOCOL = scheme + "://"
self.urls = self.__get_urls(host)
response = {}
self.HEADER['user-agent'] = self.reader.get_random_user_agent()
log.info("user-agent : " + self.HEADER['user-agent'])
log.info('Thread num : ' + str(self.threads))
try:
httplib.HTTPConnection.debuglevel = self.debug
if hasattr(urllib3, 'disable_warnings'):
urllib3.disable_warnings()
if scheme == "http":
self.http = urllib3.HTTPConnectionPool(
host.split(':')[0],
port=80 if len(host.split(':')) == 1 else int(
host.split(':')[1]),
block=True,
maxsize=10)
elif scheme == "https":
self.http = urllib3.HTTPSConnectionPool(
host.split(':')[0],
port=443 if len(host.split(':')) == 1 else int(
host.split(':')[1]),
block=True,
maxsize=10)
else:
log.critical("not support http protocl, Exit now ")
sys.exit(1)
pool = threadpool.ThreadPool(self.threads)
requests = threadpool.makeRequests(self.request, self.urls)
for req in requests:
pool.putRequest(req)
time.sleep(1)
pool.wait()
except exceptions.AttributeError as e:
log.critical(e.message)
except KeyboardInterrupt:
log.warning('Session canceled')
sys.exit()
self.counter['total'] = self.urls.__len__()
self.counter['pools'] = pool.workers.__len__()
response['count'] = self.counter
response['result'] = self.result
return response
def request(ip_address, url, header, payload=None, method='GET'):
""" Returns status and data """
pool = urllib3.HTTPSConnectionPool(ip_address, port=443, cert_reqs='CERT_NONE',
assert_hostname=False)
request_obj = pool.urlopen(method, url, headers=header, body=json.dumps(payload))
data = None
if request_obj.data and request_obj.status != 400:
data = json.loads(request_obj.data)
else:
data = request_obj.data
return request_obj.status, data
def login_by_email(email, pwd):
DEFAULT_HEADERS = {
'Host': 'github.com',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Referer': 'https://github.com/',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.8',
}
# http = ProxyManager('https://127.0.0.1:8080')
http = urllib3.HTTPSConnectionPool('github.com')
r = http.request('GET',
'https://github.com/login',
headers=DEFAULT_HEADERS)
text = r.data.decode('utf-8')
token = re.findall(r'name="authenticity_token".*?value="(.*?)"', text)[0]
cookie_login = r.headers['set-cookie']
data = {
'commit': 'Sign in',
'utf8': '✓',
'authenticity_token': token,
'login': email,
'password': pwd,
}
DEFAULT_HEADERS['Cookie'] = cookie_login
r = http.request('POST',
'https://github.com/session',
data,
headers=DEFAULT_HEADERS,
encode_multipart=False,
redirect=False)
#redirect=False取消自动跳转
cookie_session = r.headers['set-cookie']
# print(cookie_session)
DEFAULT_HEADERS['Cookie'] = cookie_session
r = http.request('GET', 'https://github.com/', headers=DEFAULT_HEADERS)
text = r.data.decode('UTF-8')
# print(text)
if 'Start a project' in text:
print('登陆成功')
# print(text)
else:
print(r.status)
print(text)
print('登陆失败')
def __init__(self, config_path, bcfslog):
# Config file
self.config_path = config_path
# Logger
self.bcfslog = bcfslog
self.bcfslog.debug("Initialise filesystem")
try:
with open(self.config_path, 'r') as config_file:
self.config = json.load(config_file)
except:
self.bcfslog.error(
"Exception Could not find configuration file: " +
self.config_path)
sys.exit("Could not find configuration file: " + self.config_path)
# Set configuation variables
self.api_url = self.config['api_url'].encode('utf-8')
self.client_id = self.config['client_id'].encode('utf-8')
self.secret = self.config['secret'].encode('utf-8')
self.redirect_url = self.config['redirect_url'].encode('utf-8')
self.auth_token = self.config['auth_token'].encode('utf-8')
self.access_token = self.config['access_token'].encode('utf-8')
# See if we need our tokens
if (self.auth_token == "") or (self.access_token == ""):
return self.authenticate()
# Start HTTP Server
self.httpd_thread = bcfsHTTPthread(
self.config['http_ip'].encode('utf-8'),
self.config['http_port'].encode('utf-8'), bcfslog, self)
self.httpd_thread.start()
# Initiate Connections
self.api_host = urlparse(self.api_url).hostname
self.api_path = urlparse(self.api_url).path
#urllib3.connection.HTTPSConnection.default_socket_options + [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
#urllib3.connection.HTTPConnection.default_socket_options + [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
# Create Connection Pool
self.bcfslog.debug("create connection pool")
self.httpsPool = urllib3.HTTPSConnectionPool(self.api_host,
maxsize=self.NUM_SOCKETS,
timeout=self.timeout)
if self.bufferAhead == 0:
self.bcfslog.debug("create single connection")
self.singleConn = requests.Session()
self.singleConn.mount(
'https://',
requests.adapters.HTTPAdapter(pool_connections=100,
pool_maxsize=100))
response = self.singleConn.get(self.api_url, verify=True)
self.bcfslog.debug("create single connection, result code:" +
str(response.status_code))
result = response.text
self.bcfslog.debug("create single connection, result text:" +
str(result))
def _api_call(self, url, method='GET', fields=None, headers={}):
self.log('_api_call(' + method + ', ' + url + ')')
auth = bytes.decode(
base64.b64encode((self.username + ':' + self.password).encode()))
headers['Authorization'] = 'Basic %s' % auth
headers['User-Agent'] = 'democraticd (mailto:[email protected])'
headers['Accept'] = '*/*'
last_modified_key = method + ' ' + url
if last_modified_key in self.last_modified:
headers['If-Modified-Since'] = self.last_modified[
last_modified_key]
if (fields != None) and (method in self.json_methods):
headers['Content-Type'] = 'application/x-www-form-urlencoded'
self.log(headers, DebugLevel.INFO)
if fields:
self.log(fields, DebugLevel.INFO)
if not self.conn_pool:
self.conn_pool = urllib3.HTTPSConnectionPool(self.base_url,
maxsize=1)
if method in self.json_methods:
body = None
if fields:
body = json.dumps(fields)
result = self.conn_pool.urlopen(method,
url,
headers=headers,
body=body)
else:
result = self.conn_pool.request_encode_url(method,
url,
headers=headers,
fields=fields)
if not result:
raise GitHubError('urllib request did not return a result')
if result.headers.get('last-modified'):
self.last_modified[last_modified_key] = result.headers.get(
'last-modified')
self.log(result.headers, DebugLevel.INFO)
if result.data:
self.log(
'result.data:\n' +
json.dumps(json.loads(result.data.decode('utf-8')),
sort_keys=True,
indent=4), DebugLevel.DEBUG)
return result
def get_json(node):
"""Send HTTP GET request to download JSON list."""
try:
http = urllib3.HTTPSConnectionPool(node, 443, timeout=10,
cert_reqs='CERT_NONE', assert_hostname=False)
response = http.request('GET', "/antanistaticmap/stats/yesterday")
except Exception as error:
print error
return ""
if response.status == 200:
return response.data
else:
return ""
def __init__(self, host, proto="http", port=80, timeout=15, strict=True, retries=None, redirect=True,
assert_same_host=False, assert_hostname=None, assert_fingerprint=None, ssl_version=None):
if "://" in host:
host = host.split("://")[1].split("/")[0]
socket.gethostbyname(host)
self.request_cfg = {"timeout": timeout, "retries": retries, "redirect": redirect, "assert_same_host": assert_same_host}
if proto.lower() == "https":
self.connection = urllib3.HTTPSConnectionPool(host, port=port, timeout=timeout, retries=retries, ssl_version=ssl_version, assert_hostname=assert_hostname, assert_fingerprint=assert_fingerprint)
else:
self.connection = urllib3.HTTPConnectionPool(host, port=port, timeout=timeout, retries=retries, strict=strict)
self.target = host
def test_urlfetch_called_with_https(self):
"""
Check that URLFetch is used when fetching https resources
"""
resp = MockResponse('OK', 200, False, 'https://www.google.com',
{'content-type': 'text/plain'})
fetch_patch = patch('google.appengine.api.urlfetch.fetch',
return_value=resp)
with fetch_patch as fetch_mock:
import urllib3
pool = urllib3.HTTPSConnectionPool('www.google.com', '443')
pool.ConnectionCls = urllib3.connection.UnverifiedHTTPSConnection
r = pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
self.assertEqual(fetch_mock.call_count, 1)
def test_urlfetch_called_with_https(self):
"""
Check that URLFetch is used when fetching https resources
"""
resp = MockResponse("OK", 200, False, "https://www.google.com",
{"content-type": "text/plain"})
fetch_patch = patch("google.appengine.api.urlfetch.fetch",
return_value=resp)
with fetch_patch as fetch_mock:
import urllib3
pool = urllib3.HTTPSConnectionPool("www.google.com", "443")
pool.ConnectionCls = urllib3.connection.UnverifiedHTTPSConnection
r = pool.request("GET", "/")
assert r.status == 200, r.data
assert fetch_mock.call_count == 1
