def __init__(self, size_or_pool=1000): if isinstance(size_or_pool, GreenPool): self.pool = size_or_pool else: self.pool = GreenPool(size_or_pool) self.waiters = queue.LightQueue() self.counter = 0
def __init__(self, conf, logger=None): """ :param conf: configuration object obtained from ConfigParser :param logger: logging object """ self.conf = conf self.logger = PrefixLoggerAdapter( logger or get_logger(conf, log_route='object-replicator'), {}) self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.bind_ip = conf.get('bind_ip', '0.0.0.0') self.servers_per_port = int(conf.get('servers_per_port', '0') or 0) self.port = None if self.servers_per_port else \ int(conf.get('bind_port', 6200)) self.concurrency = int(conf.get('concurrency', 1)) self.replicator_workers = int(conf.get('replicator_workers', 0)) self.stats_interval = int(conf.get('stats_interval', '300')) self.ring_check_interval = int(conf.get('ring_check_interval', 15)) self.next_check = time.time() + self.ring_check_interval self.replication_cycle = random.randint(0, 9) self.partition_times = [] self.interval = int( conf.get('interval') or conf.get('run_pause') or 30) self.rsync_timeout = int( conf.get('rsync_timeout', DEFAULT_RSYNC_TIMEOUT)) self.rsync_io_timeout = conf.get('rsync_io_timeout', '30') self.rsync_bwlimit = conf.get('rsync_bwlimit', '0') self.rsync_compress = config_true_value( conf.get('rsync_compress', 'no')) self.rsync_module = conf.get('rsync_module', '').rstrip('/') if not self.rsync_module: self.rsync_module = '{replication_ip}::object' self.http_timeout = int(conf.get('http_timeout', 60)) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "object.recon") self._next_rcache_update = time.time() + self.stats_interval self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.node_timeout = float(conf.get('node_timeout', 10)) self.sync_method = getattr(self, conf.get('sync_method') or 'rsync') self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.default_headers = { 'Content-Length': '0', 'user-agent': 'object-replicator %s' % os.getpid() } self.rsync_error_log_line_length = \ int(conf.get('rsync_error_log_line_length', 0)) self.handoffs_first = config_true_value( conf.get('handoffs_first', False)) self.handoff_delete = config_auto_int_value( conf.get('handoff_delete', 'auto'), 0) if any((self.handoff_delete, self.handoffs_first)): self.logger.warning('Handoff only mode is not intended for normal ' 'operation, please disable handoffs_first and ' 'handoff_delete before the next ' 'normal rebalance') self.is_multiprocess_worker = None self._df_router = DiskFileRouter(conf, self.logger) self._child_process_reaper_queue = queue.LightQueue()
def Run(self, context, *args, **kwargs): '''set up queue, start fetchers''' if self._input_queue == None: LOG.info(_("Manager %(cname)s inputqueue is None"), {'cname': self.__class__.__name__}) raise for fetcher in self.fetcher_managers: fetcher.m_output_queue = self.output_queue fetcher.m_input_queue = queue.LightQueue(CONF.max_queue_size) context.tg.add_thread(fetcher.Run, context, *args, **kwargs) while True: if self._input_queue.empty(): greenthread.sleep(3) continue crawldoc = self._input_queue.get() fetch_id = self.dispatcher.dispatch(crawldoc) print fetch_id print len(self.fetcher_managers) LOG.debug( _("Prepare dispatch to %(fetch_num)s , crawldoc: %(crawldoc)s" ), { 'fetch_num': self.fetcher_managers[fetch_id].m_id, 'crawldoc': crawldoc }) '''TODO: One fetcher hang will hang dispatch, if queue full, try put the queue front element to queue end''' self.fetcher_managers[fetch_id].wait_for_inputqueue_ready() self.fetcher_managers[fetch_id].m_input_queue.put(crawldoc)
def perform_requests(operator, nodes, element, cfg, inputdata): cryptit = cfg.decrypt cfg.decrypt = True configdata = cfg.get_node_attributes(nodes, _configattributes) cfg.decrypt = cryptit resultdata = queue.LightQueue() livingthreads = set([]) for node in nodes: livingthreads.add( _ipmiworkers.spawn(perform_request, operator, node, element, configdata, inputdata, cfg, resultdata)) while livingthreads: try: datum = resultdata.get(timeout=10) while datum: if datum != 'Done': yield datum datum = resultdata.get_nowait() except queue.Empty: pass for t in list(livingthreads): if t.dead: livingthreads.discard(t) try: # drain queue if a thread put something on the queue and died while True: datum = resultdata.get_nowait() if datum != 'Done': yield datum except queue.Empty: pass
def __init__(self, max_workers=1000): assert int(max_workers) > 0, 'Max workers must be greater than zero' self._max_workers = int(max_workers) self._pool = greenpool.GreenPool(self._max_workers) self._work_queue = queue.LightQueue() self._shutdown_lock = threading.RLock() self._shutdown = False
def __init__(self, max_workers=1000): assert EVENTLET_AVAILABLE, 'eventlet is needed to use GreenExecutor' assert int(max_workers) > 0, 'Max workers must be greater than zero' self._max_workers = int(max_workers) self._pool = greenpool.GreenPool(self._max_workers) self._work_queue = queue.LightQueue() self._shutdown_lock = threading.RLock() self._shutdown = False
def __init__(self, conf, msg_id, timeout, connection_pool): self._msg_id = msg_id self._timeout = timeout or conf.rpc_response_timeout self._reply_proxy = connection_pool.reply_proxy self._done = False self._got_ending = False self._conf = conf self._dataqueue = queue.LightQueue() # Add this caller to the reply proxy's call_waiters self._reply_proxy.add_call_waiter(self, self._msg_id)
def perform_requests(operator, nodes, element, cfg, inputdata): cryptit = cfg.decrypt cfg.decrypt = True configdata = cfg.get_node_attributes(nodes, _configattributes) cfg.decrypt = cryptit resultdata = queue.LightQueue() pendingnum = len(nodes) for node in nodes: _ipmiworkers.spawn_n( perform_request, operator, node, element, configdata, inputdata, cfg, resultdata) while pendingnum: datum = resultdata.get() if datum == 'Done': pendingnum -= 1 else: yield datum
def retrieve(nodes, element, configmanager, inputdata): results = queue.LightQueue() workers = set([]) if element == ['power', 'state']: for node in nodes: yield msg.PowerState(node=node, state='on') return elif element == ['health', 'hardware']: _run_method(retrieve_health, workers, results, configmanager, nodes, element) elif element[:3] == ['inventory', 'hardware', 'all']: _run_method(retrieve_inventory, workers, results, configmanager, nodes, element) elif element[:3] == ['inventory', 'firmware', 'all']: _run_method(retrieve_firmware, workers, results, configmanager, nodes, element) elif element == ['sensors', 'hardware', 'all']: _run_method(list_sensors, workers, results, configmanager, nodes, element) elif element[:3] == ['sensors', 'hardware', 'all']: _run_method(retrieve_sensors, workers, results, configmanager, nodes, element) else: for node in nodes: yield msg.ConfluentNodeError(node, 'Not Implemented') return while workers: try: datum = results.get(10) while datum: if datum: yield datum datum = results.get_nowait() except queue.Empty: pass eventlet.sleep(0.001) for t in list(workers): if t.dead: workers.discard(t) try: while True: datum = results.get_nowait() if datum: yield datum except queue.Empty: pass
def retrieve(nodes, element, configmanager, inputdata): results = queue.LightQueue() workers = set([]) if element == ['power', 'state']: for node in nodes: yield msg.PowerState(node=node, state='on') return elif element == ['health', 'hardware']: creds = configmanager.get_node_attributes(nodes, [ 'secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword' ], decrypt=True) for node in nodes: workers.add( eventlet.spawn(retrieve_health, configmanager, creds, node, results)) else: for node in nodes: yield msg.ConfluentNodeError(node, 'Not Implemented') return currtimeout = 10 while workers: try: datum = results.get(10) while datum: if datum: yield datum datum = results.get_nowait() except queue.Empty: pass eventlet.sleep(0.001) for t in list(workers): if t.dead: workers.discard(t) try: while True: datum = results.get_nowait() if datum: yield datum except queue.Empty: pass
def __init__(self, min_size=0, max_size=4, order_as_stack=False, create=None): """*order_as_stack* governs the ordering of the items in the free pool. If ``False`` (the default), the free items collection (of items that were created and were put back in the pool) acts as a round-robin, giving each item approximately equal utilization. If ``True``, the free pool acts as a FILO stack, which preferentially re-uses items that have most recently been used. """ self.min_size = min_size self.max_size = max_size self.order_as_stack = order_as_stack self.current_size = 0 self.channel = queue.LightQueue(0) self.free_items = collections.deque() if create is not None: self.create = create for x in xrange(min_size): self.current_size += 1 self.free_items.append(self.create())
def __init__(self, size_or_pool): super(GreenMap, self).__init__(size_or_pool) self.waiters = queue.LightQueue(maxsize=self.pool.size)
def __init__(self): """Create a :class:`SequentialEventletHandler` instance""" self.callback_queue = green_queue.LightQueue() self.completion_queue = green_queue.LightQueue() self._workers = [] self._started = False
def init_pool(self): self.pool = queue.LightQueue(0)
def subscribe(self, tenant): sub = queue.LightQueue() self._subscribers.setdefault(tenant, set()).add(sub) return sub