def __init__(self, size, **kwargs):
if not isinstance(size, int):
raise TypeError("Pool 'size' arg must be an integer")
if not size > 0:
raise ValueError("Pool 'size' arg must be greater than zero")
logger.debug("Initializing connection pool with %d connections", size)
self._lock = threading.Lock()
self._queue = queue.LifoQueue(maxsize=size)
self._thread_connections = threading.local()
connection_kwargs = kwargs
connection_kwargs['autoconnect'] = False
for i in range(size):
connection = Connection(**connection_kwargs)
self._queue.put(connection)
# The first connection is made immediately so that trivial
# mistakes like unresolvable host names are raised immediately.
# Subsequent connections are connected lazily.
with self.connection():
pass
def _get_traces(self, model, guide, *args, **kwargs):
"""
Runs the guide and runs the model against the guide with
the result packaged as a trace generator.
"""
if self.max_plate_nesting == float('inf'):
self._guess_max_plate_nesting(model, guide, *args, **kwargs)
if self.vectorize_particles:
guide = self._vectorized_num_particles(guide)
model = self._vectorized_num_particles(model)
# Enable parallel enumeration over the vectorized guide and model.
# The model allocates enumeration dimensions after (to the left of) the guide,
# accomplished by preserving the _ENUM_ALLOCATOR state after the guide call.
guide_enum = EnumerateMessenger(first_available_dim=-1 -
self.max_plate_nesting)
model_enum = EnumerateMessenger() # preserve _ENUM_ALLOCATOR state
guide = guide_enum(guide)
model = model_enum(model)
q = queue.LifoQueue()
guide = poutine.queue(guide,
q,
escape_fn=iter_discrete_escape,
extend_fn=iter_discrete_extend)
for i in range(1 if self.vectorize_particles else self.num_particles):
q.put(poutine.Trace())
while not q.empty():
yield self._get_trace(model, guide, *args, **kwargs)
def __init__(self, curl_factory=pycurl.Curl, maxsize=1, **kwargs):
self._block = kwargs.get("block", False)
self._pool = queue.LifoQueue(maxsize)
for _ in range(maxsize):
handler = curl_factory()
self._pool.put(handler, block=False)
def __init__(self, size, **kwargs):
if not isinstance(size, int):
raise TypeError("Pool 'size' arg must be an integer")
if not size > 0:
raise ValueError("Pool 'size' arg must be greater than zero")
logger.debug("Initializing connection pool with %d connections", size)
self._lock = threading.Lock()
self._queue = queue.LifoQueue(maxsize=size)
self._thread_connections = threading.local()
connection_kwargs = kwargs
connection_kwargs['autoconnect'] = False
for i in range(size):
connection = KerberosConnection(**connection_kwargs)
self._queue.put(connection)
# The first connection is made immediately so that trivial
# mistakes like unresolvable host names are raised immediately.
# Subsequent connections are connected lazily.
with self.connection():
pass
# keep alive in a separate thread by running conn.tables()
try:
self.thread = threading.Thread(target=self.keep_alive)
self.thread.daemon = True
self.thread.start()
except (KeyboardInterrupt, SystemExit) as e:
pass
def __init__(self,
size=DEFAULT_SIZE,
default_timeout=DEFAULT_TIMEOUT,
labels=None):
super(FixedSizePool, self).__init__(labels=labels)
self.size = size
self.default_timeout = default_timeout
self._sessions = queue.LifoQueue(size)
def __init__(self, settings):
# self.options = settings.get('SELENIUM_OPTIONS', {})
max_run = settings.get('SELENIUM_MAXRUN', 10)
self.sem = defer.DeferredSemaphore(max_run)
self.queue = queue.LifoQueue(max_run)
SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed)
def __init__(self, settings):
self.options = settings.get('SELENIUM_OPTIONS', {})
self.domain_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_IP')
max_run = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
logging.info("Download workers: %s", max_run)
self.sem = defer.DeferredSemaphore(max_run)
self.queue = queue.LifoQueue(max_run)
SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed)
def __init__(self, settings):
self.settings = settings
self.options = settings.get('PHANTOMJS_OPTIONS', {})\
max_run = settings.get('PHANTOMJS_MAXRUN', 5)
self.sem = defer.DeferredSemaphore(
max_run) # as a means of limiting parallelism
self.queue = queue.LifoQueue(
max_run) # last in first out, the content is driver not request
SignalManager(dispatcher.Any).connect(self._close,
signal=signals.spider_closed)
def __init__(self, path, **kwargs):
"""
Initialize the tile class. See the base class for other available
parameters.
:param path: a filesystem path for the tile source.
"""
super(OpenjpegFileTileSource, self).__init__(path, **kwargs)
largeImagePath = self._getLargeImagePath()
self._largeImagePath = largeImagePath
self._pixelInfo = {}
try:
self._openjpeg = glymur.Jp2k(largeImagePath)
except glymur.jp2box.InvalidJp2kError:
raise TileSourceException(
'File cannot be opened via Glymur and OpenJPEG.')
self._openjpegHandles = queue.LifoQueue()
for _ in range(self._maxOpenHandles - 1):
self._openjpegHandles.put(None)
self._openjpegHandles.put(self._openjpeg)
try:
self.sizeY, self.sizeX = self._openjpeg.shape[:2]
except IndexError:
raise TileSourceException(
'File cannot be opened via Glymur and OpenJPEG.')
self.levels = int(self._openjpeg.codestream.segment[2].num_res) + 1
self._minlevel = 0
self.tileWidth = self.tileHeight = 2**int(
math.ceil(
max(
math.log(float(self.sizeX)) / math.log(2) - self.levels +
1,
math.log(float(self.sizeY)) / math.log(2) - self.levels +
1)))
# Small and large tiles are both inefficient. Large tiles don't work
# with some viewers (leaflet and Slide Atlas, for instance)
if self.tileWidth < self._minTileSize or self.tileWidth > self._maxTileSize:
self.tileWidth = self.tileHeight = min(
self._maxTileSize, max(self._minTileSize, self.tileWidth))
self.levels = int(
math.ceil(
math.log(
float(max(self.sizeX, self.sizeY)) / self.tileWidth) /
math.log(2))) + 1
self._minlevel = self.levels - self._openjpeg.codestream.segment[
2].num_res - 1
self._getAssociatedImages()
def __init__(self):
# get_ip = Get_ip()
# proxy = get_ip.random_ip()
ua = UserAgent()
headers = ua.random
print(headers)
dcap = dict(DesiredCapabilities.CHROME)
dcap["Chrome.page.settings.loadImages"] = True
dcap["Chrome.page.settings.userAgent"] = headers
# chromeOptions.add_argument('--proxy-server="{0}"'.format(proxy))
self.Chromeoptions = webdriver.ChromeOptions()
self.Chromeoptions.add_argument('--headless')
self.sem = defer.DeferredSemaphore(10)
self.queue = queue.LifoQueue(self.Chromeoptions)
SignalManager(dispatcher.Any).connect(self._close,
signal=signals.spider_closed)
def __init__(self, size, hosts=None, **kwargs):
'''
hosts:
A list of hosts or a string of hosts seperated by ","
This parameter works only if host is not specified
'''
if not isinstance(size, int):
raise TypeError("Pool 'size' arg must be an integer")
if not size > 0:
raise ValueError("Pool 'size' arg must be greater than zero")
logger.debug(
"Initializing connection pool with %d connections", size)
self._lock = threading.Lock()
self._host_queue_map = {}
self._thread_connections = threading.local()
connection_kwargs = kwargs
connection_kwargs['autoconnect'] = False
if kwargs.get('host'):
self._hosts = [kwargs.get('host')]
else:
if isinstance(hosts, list):
self._hosts = hosts
elif isinstance(hosts, six.text_type):
self._hosts = hosts.split(',')
else:
raise Exception('error hosts type')
for host in self._hosts:
self._host_queue_map[host] = queue.LifoQueue(maxsize=size)
connection_kwargs['host'] = host
for i in range(size):
connection = KerberosConnection(**connection_kwargs)
self._host_queue_map[host].put(connection)
# self._queue.put(connection)
# The first connection is made immediately so that trivial
# mistakes like unresolvable host names are raised immediately.
# Subsequent connections are connected lazily.
with self.connection():
pass
def __init__(self, size, hosts=[], **kwargs):
'''
hosts: 格式为列表或者逗号隔开的字符串；host存在时，以host为准；host为空时，hosts才会生效。
'''
if not isinstance(size, int):
raise TypeError("Pool 'size' arg must be an integer")
if not size > 0:
raise ValueError("Pool 'size' arg must be greater than zero")
logger.debug("Initializing connection pool with %d connections", size)
self._lock = threading.Lock()
self._host_queue_map = {}
self._thread_connections = threading.local()
connection_kwargs = kwargs
connection_kwargs['autoconnect'] = False
if kwargs.get('host'):
self._hosts = [kwargs.get('host')]
else:
if isinstance(hosts, list):
self._hosts = hosts
elif isinstance(hosts, str) or isinstance(hosts, unicode):
self._hosts = hosts.split(',')
else:
raise Exception('error hosts type')
for host in self._hosts:
self._host_queue_map[host] = queue.LifoQueue(maxsize=size)
connection_kwargs['host'] = host
for i in range(size):
connection = KerberosConnection(**connection_kwargs)
self._host_queue_map[host].put(connection)
# self._queue.put(connection)
# The first connection is made immediately so that trivial
# mistakes like unresolvable host names are raised immediately.
# Subsequent connections are connected lazily.
with self.connection():
pass
def __init__(self,
host=settings.CLICKHOUSE_SERVER.split(':')[0],
port=int(settings.CLICKHOUSE_SERVER.split(':')[1]),
connect_timeout=1,
send_receive_timeout=300,
max_pool_size=settings.CLICKHOUSE_MAX_POOL_SIZE,
client_settings={},
metrics=None,
):
self.host = host
self.port = port
self.connect_timeout = connect_timeout
self.send_receive_timeout = send_receive_timeout
self.client_settings = client_settings
self.metrics = metrics
self.pool = queue.LifoQueue(max_pool_size)
# Fill the queue up so that doing get() on it will block properly
for _ in range(max_pool_size):
self.pool.put(None)
def __init__(self, target_size=10, labels=None):
super(BurstyPool, self).__init__(labels=labels)
self.target_size = target_size
self._database = None
self._sessions = queue.LifoQueue(target_size)
def __init__(self, settings):
self.options = settings.get('PHANTOMJS_OPTIONS', {})
max_run = settings.get('PHANTOMJS_MAXRUN', 10)
self.sem = defer.DeferredSemaphore(max_run)
self.queue = queue.LifoQueue(max_run)
