def __init__(self, size, **kwargs): if not isinstance(size, int): raise TypeError("Pool 'size' arg must be an integer") if not size > 0: raise ValueError("Pool 'size' arg must be greater than zero") logger.debug("Initializing connection pool with %d connections", size) self._lock = threading.Lock() self._queue = queue.LifoQueue(maxsize=size) self._thread_connections = threading.local() connection_kwargs = kwargs connection_kwargs['autoconnect'] = False for i in range(size): connection = Connection(**connection_kwargs) self._queue.put(connection) # The first connection is made immediately so that trivial # mistakes like unresolvable host names are raised immediately. # Subsequent connections are connected lazily. with self.connection(): pass
def _get_traces(self, model, guide, *args, **kwargs): """ Runs the guide and runs the model against the guide with the result packaged as a trace generator. """ if self.max_plate_nesting == float('inf'): self._guess_max_plate_nesting(model, guide, *args, **kwargs) if self.vectorize_particles: guide = self._vectorized_num_particles(guide) model = self._vectorized_num_particles(model) # Enable parallel enumeration over the vectorized guide and model. # The model allocates enumeration dimensions after (to the left of) the guide, # accomplished by preserving the _ENUM_ALLOCATOR state after the guide call. guide_enum = EnumerateMessenger(first_available_dim=-1 - self.max_plate_nesting) model_enum = EnumerateMessenger() # preserve _ENUM_ALLOCATOR state guide = guide_enum(guide) model = model_enum(model) q = queue.LifoQueue() guide = poutine.queue(guide, q, escape_fn=iter_discrete_escape, extend_fn=iter_discrete_extend) for i in range(1 if self.vectorize_particles else self.num_particles): q.put(poutine.Trace()) while not q.empty(): yield self._get_trace(model, guide, *args, **kwargs)
def __init__(self, curl_factory=pycurl.Curl, maxsize=1, **kwargs): self._block = kwargs.get("block", False) self._pool = queue.LifoQueue(maxsize) for _ in range(maxsize): handler = curl_factory() self._pool.put(handler, block=False)
def __init__(self, size, **kwargs): if not isinstance(size, int): raise TypeError("Pool 'size' arg must be an integer") if not size > 0: raise ValueError("Pool 'size' arg must be greater than zero") logger.debug("Initializing connection pool with %d connections", size) self._lock = threading.Lock() self._queue = queue.LifoQueue(maxsize=size) self._thread_connections = threading.local() connection_kwargs = kwargs connection_kwargs['autoconnect'] = False for i in range(size): connection = KerberosConnection(**connection_kwargs) self._queue.put(connection) # The first connection is made immediately so that trivial # mistakes like unresolvable host names are raised immediately. # Subsequent connections are connected lazily. with self.connection(): pass # keep alive in a separate thread by running conn.tables() try: self.thread = threading.Thread(target=self.keep_alive) self.thread.daemon = True self.thread.start() except (KeyboardInterrupt, SystemExit) as e: pass
def __init__(self, size=DEFAULT_SIZE, default_timeout=DEFAULT_TIMEOUT, labels=None): super(FixedSizePool, self).__init__(labels=labels) self.size = size self.default_timeout = default_timeout self._sessions = queue.LifoQueue(size)
def __init__(self, settings): # self.options = settings.get('SELENIUM_OPTIONS', {}) max_run = settings.get('SELENIUM_MAXRUN', 10) self.sem = defer.DeferredSemaphore(max_run) self.queue = queue.LifoQueue(max_run) SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed)
def __init__(self, settings): self.options = settings.get('SELENIUM_OPTIONS', {}) self.domain_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self.ip_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_IP') max_run = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency logging.info("Download workers: %s", max_run) self.sem = defer.DeferredSemaphore(max_run) self.queue = queue.LifoQueue(max_run) SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed)
def __init__(self, settings): self.settings = settings self.options = settings.get('PHANTOMJS_OPTIONS', {})\ max_run = settings.get('PHANTOMJS_MAXRUN', 5) self.sem = defer.DeferredSemaphore( max_run) # as a means of limiting parallelism self.queue = queue.LifoQueue( max_run) # last in first out, the content is driver not request SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed)
def __init__(self, path, **kwargs): """ Initialize the tile class. See the base class for other available parameters. :param path: a filesystem path for the tile source. """ super(OpenjpegFileTileSource, self).__init__(path, **kwargs) largeImagePath = self._getLargeImagePath() self._largeImagePath = largeImagePath self._pixelInfo = {} try: self._openjpeg = glymur.Jp2k(largeImagePath) except glymur.jp2box.InvalidJp2kError: raise TileSourceException( 'File cannot be opened via Glymur and OpenJPEG.') self._openjpegHandles = queue.LifoQueue() for _ in range(self._maxOpenHandles - 1): self._openjpegHandles.put(None) self._openjpegHandles.put(self._openjpeg) try: self.sizeY, self.sizeX = self._openjpeg.shape[:2] except IndexError: raise TileSourceException( 'File cannot be opened via Glymur and OpenJPEG.') self.levels = int(self._openjpeg.codestream.segment[2].num_res) + 1 self._minlevel = 0 self.tileWidth = self.tileHeight = 2**int( math.ceil( max( math.log(float(self.sizeX)) / math.log(2) - self.levels + 1, math.log(float(self.sizeY)) / math.log(2) - self.levels + 1))) # Small and large tiles are both inefficient. Large tiles don't work # with some viewers (leaflet and Slide Atlas, for instance) if self.tileWidth < self._minTileSize or self.tileWidth > self._maxTileSize: self.tileWidth = self.tileHeight = min( self._maxTileSize, max(self._minTileSize, self.tileWidth)) self.levels = int( math.ceil( math.log( float(max(self.sizeX, self.sizeY)) / self.tileWidth) / math.log(2))) + 1 self._minlevel = self.levels - self._openjpeg.codestream.segment[ 2].num_res - 1 self._getAssociatedImages()
def __init__(self): # get_ip = Get_ip() # proxy = get_ip.random_ip() ua = UserAgent() headers = ua.random print(headers) dcap = dict(DesiredCapabilities.CHROME) dcap["Chrome.page.settings.loadImages"] = True dcap["Chrome.page.settings.userAgent"] = headers # chromeOptions.add_argument('--proxy-server="{0}"'.format(proxy)) self.Chromeoptions = webdriver.ChromeOptions() self.Chromeoptions.add_argument('--headless') self.sem = defer.DeferredSemaphore(10) self.queue = queue.LifoQueue(self.Chromeoptions) SignalManager(dispatcher.Any).connect(self._close, signal=signals.spider_closed)
def __init__(self, size, hosts=None, **kwargs): ''' hosts: A list of hosts or a string of hosts seperated by "," This parameter works only if host is not specified ''' if not isinstance(size, int): raise TypeError("Pool 'size' arg must be an integer") if not size > 0: raise ValueError("Pool 'size' arg must be greater than zero") logger.debug( "Initializing connection pool with %d connections", size) self._lock = threading.Lock() self._host_queue_map = {} self._thread_connections = threading.local() connection_kwargs = kwargs connection_kwargs['autoconnect'] = False if kwargs.get('host'): self._hosts = [kwargs.get('host')] else: if isinstance(hosts, list): self._hosts = hosts elif isinstance(hosts, six.text_type): self._hosts = hosts.split(',') else: raise Exception('error hosts type') for host in self._hosts: self._host_queue_map[host] = queue.LifoQueue(maxsize=size) connection_kwargs['host'] = host for i in range(size): connection = KerberosConnection(**connection_kwargs) self._host_queue_map[host].put(connection) # self._queue.put(connection) # The first connection is made immediately so that trivial # mistakes like unresolvable host names are raised immediately. # Subsequent connections are connected lazily. with self.connection(): pass
def __init__(self, size, hosts=[], **kwargs): ''' hosts: 格式为列表或者逗号隔开的字符串;host存在时,以host为准;host为空时,hosts才会生效。 ''' if not isinstance(size, int): raise TypeError("Pool 'size' arg must be an integer") if not size > 0: raise ValueError("Pool 'size' arg must be greater than zero") logger.debug("Initializing connection pool with %d connections", size) self._lock = threading.Lock() self._host_queue_map = {} self._thread_connections = threading.local() connection_kwargs = kwargs connection_kwargs['autoconnect'] = False if kwargs.get('host'): self._hosts = [kwargs.get('host')] else: if isinstance(hosts, list): self._hosts = hosts elif isinstance(hosts, str) or isinstance(hosts, unicode): self._hosts = hosts.split(',') else: raise Exception('error hosts type') for host in self._hosts: self._host_queue_map[host] = queue.LifoQueue(maxsize=size) connection_kwargs['host'] = host for i in range(size): connection = KerberosConnection(**connection_kwargs) self._host_queue_map[host].put(connection) # self._queue.put(connection) # The first connection is made immediately so that trivial # mistakes like unresolvable host names are raised immediately. # Subsequent connections are connected lazily. with self.connection(): pass
def __init__(self, host=settings.CLICKHOUSE_SERVER.split(':')[0], port=int(settings.CLICKHOUSE_SERVER.split(':')[1]), connect_timeout=1, send_receive_timeout=300, max_pool_size=settings.CLICKHOUSE_MAX_POOL_SIZE, client_settings={}, metrics=None, ): self.host = host self.port = port self.connect_timeout = connect_timeout self.send_receive_timeout = send_receive_timeout self.client_settings = client_settings self.metrics = metrics self.pool = queue.LifoQueue(max_pool_size) # Fill the queue up so that doing get() on it will block properly for _ in range(max_pool_size): self.pool.put(None)
def __init__(self, target_size=10, labels=None): super(BurstyPool, self).__init__(labels=labels) self.target_size = target_size self._database = None self._sessions = queue.LifoQueue(target_size)
def __init__(self, settings): self.options = settings.get('PHANTOMJS_OPTIONS', {}) max_run = settings.get('PHANTOMJS_MAXRUN', 10) self.sem = defer.DeferredSemaphore(max_run) self.queue = queue.LifoQueue(max_run)