def __init__(self, port):
self.server_host = 'http://cs110-proxy.jerrycainjr.com'
self.blocked_host = 'http://blacklisted.cs110-proxy.jerrycainjr.com'
self.proxy = 'localhost:%d' % port
self.curlshare = pycurl.CurlShare() # Needed to share cookies across threads
self.curlshare.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
def test_share_cycle(self):
gc.collect()
self.maybe_enable_debug()
share = pycurl.CurlShare()
t = []
searches = []
for a in range(100):
curl = util.DefaultCurl()
curl.setopt(curl.SHARE, share)
t.append(curl)
c_id = id(curl)
searches.append(c_id)
m_id = id(share)
searches.append(m_id)
self.maybe_print_objects()
del curl
del t
del share
self.maybe_print_objects()
gc.collect()
self.maybe_print_objects()
objects = gc.get_objects()
for search in searches:
for object in objects:
assert search != id(object)
def reset_session(self):
cs = pycurl.CurlShare()
cs.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
cs.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self._curlshare = cs
curl = self._new_request_worker()
setattr(self._curltls, 'curl', curl)
def __init__(self, **param):
self.buf = io.BytesIO()
share = pycurl.CurlShare()
#多个Curl可以共享cookie
share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
post_c = pycurl.Curl()
post_c.setopt(pycurl.SHARE, share)
post_c = self._init(post_c, param)
post_c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
self.post_c = post_c
get_c = pycurl.Curl()
get_c.setopt(pycurl.SHARE, share)
get_c = self._init(get_c, param)
get_c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
self.get_c = get_c
down_c = pycurl.Curl()
down_c.setopt(pycurl.SHARE, share)
down_c = self._init(down_c, param)
down_c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
self.down_c = down_c
upload_c = pycurl.Curl()
upload_c.setopt(pycurl.SHARE, share)
upload_c = self._init(upload_c, param)
upload_c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
upload_c.setopt(pycurl.TIMEOUT, 300)
self.upload_c = upload_c
self.param = param
self.share = share
def test_pickle_dumps_share(self):
s = pycurl.CurlShare()
try:
pickle.dumps(s)
except (pickle.PicklingError, TypeError):
pass
else:
self.fail("No exception when trying to pickle a CurlShare handle")
def test_copy_share(self):
s = pycurl.CurlShare()
try:
copy.copy(s)
except (copy.Error, TypeError):
pass
else:
assert False, "No exception when trying to copy a CurlShare handle"
def __init__(self):
self.curl = pycurl.Curl()
self.curls = [] #for multicurl
self.buffers = []
self.s = pycurl.CurlShare()
self.s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self.s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
self.s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
def __init__(self, payload=None, share=None):
self.payload = payload
if share:
s = pycurl.CurlShare()
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
self.share = s
else:
self.share = None
def test_pickle_share(self):
s = pycurl.CurlShare()
fp = util.StringIO()
p = pickle.Pickler(fp, 1)
try:
p.dump(s)
except (pickle.PicklingError, TypeError):
pass
else:
assert 0, "No exception when trying to pickle a CurlShare handle"
del s, fp, p
def __init__(self, max_query=10, loop=get_event_loop()):
self.loop = loop
self.multi = pycurl.CurlMulti()
self.share = pycurl.CurlShare()
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
self.handles = set()
self.waiting_handles = Queue(loop=self.loop)
self.running = True
self.max_active = max_query
def initialize(self, *args, **kwargs):
super(TornadoHTTPClient, self).initialize(*args, **kwargs)
self._cookie = {}
self._proxy = {}
self._user_agent = None
self.keep_alive = True
self.use_cookie = True
self.debug = False
self.validate_cert = True
self._headers = {}
self._share = pycurl.CurlShare()
self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
self.setup_curl()
def test_share(self):
s = pycurl.CurlShare()
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
t1 = WorkerThread(s)
t2 = WorkerThread(s)
t1.start()
t2.start()
t1.join()
t2.join()
del s
self.assertEqual('success', t1.sio.getvalue())
self.assertEqual('success', t2.sio.getvalue())
def __init__(self,
auth=None,
proxy=None,
additional_curl_options=None,
use_shared_handle=False):
self._curl = pycurl.Curl()
if use_shared_handle:
###use shared Cookie, DNS and SSL Session ID caches when operating multi-threaded
shared_curl = pycurl.CurlShare()
shared_curl.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
shared_curl.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
shared_curl.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
self._curl.setopt(pycurl.SHARE, shared_curl)
if additional_curl_options:
self._additional_curl_options = additional_curl_options
else:
self._additional_curl_options = {}
if auth:
auth.configure_auth(self._curl)
if proxy:
proxy.configure_proxy(self._curl)
def __init__(self, base="http://localhost:9200", debug_filter=CurlDebugType.HEADER + CurlDebugType.DATA, verify_certs=True, verbose=False, ca_file=None, kerberos=None, ssl_opts={}, user_agent=None):
print('init ElasticConnection')
self._curl = pycurl.CurlMulti()
# Activate share settings
self._share = pycurl.CurlShare()
self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
self._base = base
self.debug_filter = debug_filter
self.handles = set()
self.waiting_handles = queue.Queue(1000)
self.verify_certs = verify_certs
self.ca_file = ca_file
self.verbose = verbose
self.kerberos = kerberos
self.user_agent = user_agent
self.ssl_opts = {}
for k,v in ssl_opts:
if k in ElasticConnection.ssl_opts_mapping and v is not None:
self.ssl_opts[ElasticConnection.ssl_opts_mapping[k]] = v
import sys
import pycurl
import threading
print >> sys.stderr, 'Testing', pycurl.version
class Test(threading.Thread):
def __init__(self, share):
threading.Thread.__init__(self)
self.curl = pycurl.Curl()
self.curl.setopt(pycurl.URL, 'http://curl.haxx.se')
self.curl.setopt(pycurl.SHARE, share)
def run(self):
self.curl.perform()
self.curl.close()
s = pycurl.CurlShare()
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
t1 = Test(s)
t2 = Test(s)
t1.start()
t2.start()
del s
def test_keyword_arguments(self):
pycurl.CurlShare(a=1)
def test_positional_arguments(self):
pycurl.CurlShare(1)
def test_keyword_arguments(self):
with pytest.raises(TypeError):
pycurl.CurlShare(a=1)
def _new_curl_share():
curlshare = pycurl.CurlShare()
curlshare.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
curlshare.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
return curlshare
def test_share_close_twice(self):
s = pycurl.CurlShare()
s.close()
s.close()
def test_positional_arguments(self):
with pytest.raises(TypeError):
pycurl.CurlShare(1)
def test_share(self):
s = pycurl.CurlShare()
weakref.ref(s)
s.close()
from urllib import request
import json
import struct
import sys
import pycurl
from io import BytesIO
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend
aes_backend = default_backend()
AES_BLOCK_SIZE = 16
chunk_size = 16 << 20 # 16M
USING_IPV6 = True
curl_share = pycurl.CurlShare()
curl_share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
curl_share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
Cache = {}
def post(url, f=None, post_data=None, c=None):
if f is None:
buffer = BytesIO()
post(url, buffer, post_data)
return buffer.getvalue()
cleanup = False
if c is None:
c = pycurl.Curl()
c.setopt(pycurl.SHARE, curl_share)
cleanup = True
All functions needed to steer and execute downloads of web documents.
"""
import logging
import random
import re
from collections import defaultdict, deque, namedtuple
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from time import sleep
import certifi
try:
import pycurl
CURL_SHARE = pycurl.CurlShare()
# available options: https://pycurl.io/docs/latest/curlshareobject.html?highlight=lock_data_cookie
CURL_SHARE.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
CURL_SHARE.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
# not thread-safe
# https://curl.se/libcurl/c/curl_share_setopt.html
# CURL_SHARE.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_CONNECT)
except ImportError:
pycurl = None
import urllib3
from courlan import get_host_and_path, validate_url
from . import __version__
from .settings import DEFAULT_CONFIG, DOWNLOAD_THREADS, TIMEOUT
from .utils import decode_response, uniquify_list
def __init__(self):
self.shared = pycurl.CurlShare()
self.shared.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
def __init__(self):
BasicRequests.__init__(self)
self._curlshare = pycurl.CurlShare()
self._curlshare.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
self._curlshare.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
