def __init__(self, conf=cfg.CONF, ffi=None): self.enc_sem = semaphore.Semaphore() self.dec_sem = semaphore.Semaphore() self.verify_sem = semaphore.Semaphore() self.block_size = 16 # in bytes # TODO(reaperhulk): abstract this so alternate algorithms/vendors # are possible. self.algorithm = VENDOR_SAFENET_CKM_AES_GCM if conf.p11_crypto_plugin.library_path is None: raise ValueError(u._("library_path is required")) self.ffi = _build_ffi() if not ffi else ffi self.lib = self.ffi.dlopen(conf.p11_crypto_plugin.library_path) self._check_error(self.lib.C_Initialize(self.ffi.NULL)) self.session = self._open_session(1) self.rw_session = self._open_session(1) self._login(conf.p11_crypto_plugin.login, self.session) self._login(conf.p11_crypto_plugin.login, self.rw_session) self._perform_rng_self_test() self.current_mkek_label = conf.p11_crypto_plugin.mkek_label self.current_hmac_label = conf.p11_crypto_plugin.hmac_label LOG.debug("Current mkek label: %s", self.current_mkek_label) LOG.debug("Current hmac label: %s", self.current_hmac_label) self.key_handles = {} # cache current MKEK handle in the dictionary self._get_or_generate_mkek(self.current_mkek_label, conf.p11_crypto_plugin.mkek_length) self._get_or_generate_hmac_key(self.current_hmac_label)
def test_parallel_api_list_and_delete_operations(self): # One execution already exists. Let's create another one. wf_ex = self.engine.start_workflow('wf') self.await_workflow_success(wf_ex.id) self.assertEqual(2, len(db_api.get_workflow_executions())) delete_lock = semaphore.Semaphore(0) list_lock = semaphore.Semaphore(0) orig_func = execution._get_workflow_execution_resource def delete_(): context.set_ctx(unit_base.get_context()) db_api.delete_workflow_execution(self.wf_ex_id) # Unlocking the "list" operation. list_lock.release() def list_(): resp = self.app.get('/v2/executions/') self.assertEqual(1, len(resp.json['executions'])) # This decorator is needed to halt the thread of the "list" # operation and wait till the "delete" operation is over. # That way we'll reproduce the situation when the "list" # operation has already fetched the execution but it then # gets deleted before further lazy-loading of the execution # fields. def decorate_resource_function_(arg): self.decorator_call_cnt += 1 # It makes sense to use this trick only once since only # one object gets deleted. if self.decorator_call_cnt == 1: # It's OK now to delete the execution so we release # the corresponding lock. delete_lock.release() # Wait till the "delete" operation has finished. list_lock.acquire() return orig_func(arg) with mock.patch.object(execution, '_get_workflow_execution_resource', wraps=decorate_resource_function_): self.threads.append(eventlet.spawn(list_)) # Make sure that the "list" operation came to the right point # which is just about the call to the resource function. delete_lock.acquire() self.threads.append(eventlet.spawn(delete_)) for t in self.threads: t.wait()
def setUp(self): super(EngineActionRaceConditionTest, self).setUp() global ACTION_SEMAPHORE global TEST_SEMAPHORE ACTION_SEMAPHORE = semaphore.Semaphore(1) TEST_SEMAPHORE = semaphore.Semaphore(0) test_base.register_action_class('test.block', BlockingAction)
def __init__(self, sock, environ, version=76): """ :param socket: The eventlet socket :type socket: :class:`eventlet.greenio.GreenSocket` :param environ: The wsgi environment :param version: The WebSocket spec version to follow (default is 76) """ self.log = environ.get('wsgi.errors', sys.stderr) self.log_context = 'server={shost}/{spath} client={caddr}:{cport}'.format( shost=environ.get('HTTP_HOST'), spath=environ.get('SCRIPT_NAME', '') + environ.get('PATH_INFO', ''), caddr=environ.get('REMOTE_ADDR'), cport=environ.get('REMOTE_PORT'), ) self.socket = sock self.origin = environ.get('HTTP_ORIGIN') self.protocol = environ.get('HTTP_WEBSOCKET_PROTOCOL') self.path = environ.get('PATH_INFO') self.environ = environ self.version = version self.websocket_closed = False self._buf = b"" self._msgs = collections.deque() self._sendlock = semaphore.Semaphore()
def test_queue_lock_order(self): q = zmq._QueueLock() s = semaphore.Semaphore(0) results = [] def lock(x): with q: results.append(x) s.release() q.acquire() spawn(lock, 1) sleep() spawn(lock, 2) sleep() spawn(lock, 3) sleep() self.assertEqual(results, []) q.release() s.acquire() s.acquire() s.acquire() self.assertEqual(results, [1, 2, 3])
def __init__(self, user_id=None, tenant_id=None, token=None, service_catalog=None, username=None, tenant_name=None, roles=None, is_admin=None, remote_semaphore=None, auth_uri=None, **kwargs): if kwargs: LOG.warn(_LW('Arguments dropped when creating context: %s'), kwargs) self.user_id = user_id self.tenant_id = tenant_id self.token = token self.service_catalog = service_catalog self.username = username self.tenant_name = tenant_name self.is_admin = is_admin self.remote_semaphore = remote_semaphore or semaphore.Semaphore( CONF.cluster_remote_threshold) self.roles = roles if auth_uri: self.auth_uri = auth_uri else: self.auth_uri = _get_auth_uri()
def test_nested_acquire(self): q = zmq._QueueLock() self.assertFalse(q) q.acquire() q.acquire() s = semaphore.Semaphore(0) results = [] def lock(x): with q: results.append(x) s.release() spawn(lock, 1) sleep() self.assertEqual(results, []) q.release() sleep() self.assertEqual(results, []) self.assertTrue(q) q.release() s.acquire() self.assertEqual(results, [1])
def __init__(self, host, conf=None): if conf: self.conf = conf else: self.conf = cfg.CONF self.root_helper = config.get_root_helper(self.conf) self.router_info = {} if not self.conf.interface_driver: raise SystemExit(_('An interface driver must be specified')) try: self.driver = importutils.import_object(self.conf.interface_driver, self.conf) except: msg = _("Error importing interface driver " "'%s'") % self.conf.interface_driver raise SystemExit(msg) self.context = context.get_admin_context_without_session() self.plugin_rpc = L3PluginApi(topics.PLUGIN, host) self.fullsync = True self.sync_sem = semaphore.Semaphore(1) if self.conf.use_namespaces: self._destroy_router_namespaces(self.conf.router_id) super(L3NATAgent, self).__init__(host=self.conf.host)
def __init__(self, socket, signal_bus, is_reactive_conn=False): # Validate input. if socket is None: raise ValueError('Invalid arguments passed.') self._remotename = self.get_remotename(socket) self._localname = self.get_localname(socket) activity_name = ('BgpProtocol %s, %s, %s' % (is_reactive_conn, self._remotename, self._localname)) Activity.__init__(self, name=activity_name) # Intialize instance variables. self._peer = None self._recv_buff = '' self._socket = socket self._socket.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1) self._sendlock = semaphore.Semaphore() self._signal_bus = signal_bus self._holdtime = None self._keepalive = None self._expiry = None # Add socket to Activity's socket container for managing it. if is_reactive_conn: self._asso_socket_map['passive_conn'] = self._socket else: self._asso_socket_map['active_conn'] = self._socket self._open_msg = None self.state = BGP_FSM_CONNECT self._is_reactive = is_reactive_conn self.sent_open_msg = None self.recv_open_msg = None self._is_bound = False
def Semaphore(count): warnings.warn( "The Semaphore class has moved! Please " "use semaphore.Semaphore instead.", DeprecationWarning, stacklevel=2) return semaphoremod.Semaphore(count)
def inner(*args, **kwargs): # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn\ # amically-allocating-and-destroying-mutexes if name not in _semaphores: _semaphores[name] = semaphore.Semaphore() sem = _semaphores[name] LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method ' '"%(method)s"...' % {'lock': name, 'method': f.__name__})) with sem: if external: LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...' % {'lock': name, 'method': f.__name__})) lock_file_path = os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name) lock = lockfile.FileLock(lock_file_path) else: lock = _NoopContextManager() with lock: retval = f(*args, **kwargs) # If no-one else is waiting for it, delete it. # See note about possible raciness above. if not sem.balance < 1: del _semaphores[name] return retval
def setup_remote(self, engine): global _global_remote_semaphore global INFRA _global_remote_semaphore = semaphore.Semaphore( CONF.global_remote_threshold) INFRA = engine
def __init__(self, host, username, password, port=22): self.ssh_pool = self._init_ssh_pool(host, port, username, password) # Note(cfouts) Number of SSH connections made to the backend need to be # limited. Use of SSHPool allows connections to be cached and reused # instead of creating a new connection each time a command is executed # via SSH. self.ssh_connect_semaphore = semaphore.Semaphore( self.MAX_CONCURRENT_SSH_CONNECTIONS)
def inner(*args, **kwargs): # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, semaphore.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s" for method ' '"%(method)s"...'), {'lock': name, 'method': f.__name__}) if external and not FLAGS.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...'), {'lock': name, 'method': f.__name__}) cleanup_dir = False def wrap_mkdtemp(): cleanup_dir = True return tempfile.mkdtemp() # We need a copy of lock_path because it is non-local local_lock_path = lock_path if not local_lock_path: local_lock_path = FLAGS.lock_path or wrap_mkdtemp() if not os.path.exists(local_lock_path): cleanup_dir = True ensure_tree(local_lock_path) # NOTE(mikal): the lock name cannot contain directory # separators safe_name = name.replace(os.sep, '_') lock_file_path = os.path.join(local_lock_path, 'nova-%s' % safe_name) lock = InterProcessLock(lock_file_path) try: with lock: LOG.debug(_('Got file lock "%(lock)s" for ' 'method "%(method)s"...'), {'lock': name, 'method': f.__name__}) retval = f(*args, **kwargs) finally: # NOTE(vish): This removes the tempdir if we needed # to create one. This is used to cleanup # the locks left behind by unit tests. if cleanup_dir: shutil.rmtree(local_lock_path) else: retval = f(*args, **kwargs) return retval
def __init__(self): """ Initializes the resource pool. """ self._pool_lock = semaphore.Semaphore() self._pool = {} # Need a place to store error messages self._messages = []
def test_dirty_reads(self): sem1 = semaphore.Semaphore(0) sem2 = semaphore.Semaphore(0) def _run_tx1(): with db_api.transaction(): wf_ex = db_api.create_workflow_execution(WF_EXEC) # Release TX2 so it can read data. sem2.release() print("Created: %s" % wf_ex) print("Holding TX1...") sem1.acquire() print("TX1 completed.") def _run_tx2(): with db_api.transaction(): print("Holding TX2...") sem2.acquire() wf_execs = db_api.get_workflow_executions() print("Read: %s" % wf_execs) self.assertEqual(1, len(wf_execs)) # Release TX1 so it can complete. sem1.release() print("TX2 completed.") t1 = eventlet.spawn(_run_tx1) t2 = eventlet.spawn(_run_tx2) t1.wait() t2.wait() t1.kill() t2.kill()
def __init__(self, name, conf, **kwargs): self._started = False self.conf = conf self.name = name self._writer_engine = None self._reader_engine = None self._writer_maker = None self._reader_maker = None # self._session = None # self._rsession = None self.connection_kwargs = kwargs self.lock = semaphore.Semaphore()
def initialize(): global _tracelog global _bufferdaemon global _bufferlock _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen(['/opt/confluent/bin/vtbufferd'], bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE) fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL) fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_SETFL, fl | os.O_NONBLOCK)
def inner(*args, **kwargs): # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, semaphore.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem LOG.debug( _('Attempting to grab semaphore "%(lock)s" for method ' '"%(method)s"...'), { 'lock': name, 'method': f.__name__ }) with sem: LOG.debug( _('Got semaphore "%(lock)s" for method ' '"%(method)s"...'), { 'lock': name, 'method': f.__name__ }) if external and not FLAGS.disable_process_locking: LOG.debug( _('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...'), { 'lock': name, 'method': f.__name__ }) lock_file_path = os.path.join(FLAGS.lock_path, 'nova-%s' % name) lock = InterProcessLock(lock_file_path) with lock: LOG.debug( _('Got file lock "%(lock)s" for ' 'method "%(method)s"...'), { 'lock': name, 'method': f.__name__ }) retval = f(*args, **kwargs) else: retval = f(*args, **kwargs) return retval
def test_semaphore_contention(): g_mutex = semaphore.Semaphore() counts = [0, 0] def worker(no): while min(counts) < 200: with g_mutex: counts[no - 1] += 1 eventlet.sleep(0.001) t1 = eventlet.spawn(worker, no=1) t2 = eventlet.spawn(worker, no=2) eventlet.sleep(0.5) t1.kill() t2.kill() assert abs(counts[0] - counts[1]) < int(min(counts) * 0.1), counts
def __init__(self, sock, environ, version=76): """ :param socket: The eventlet socket :type socket: :class:`eventlet.greenio.GreenSocket` :param environ: The wsgi environment :param version: The WebSocket spec version to follow (default is 76) """ self.socket = sock self.origin = environ.get('HTTP_ORIGIN') self.protocol = environ.get('HTTP_WEBSOCKET_PROTOCOL') self.path = environ.get('PATH_INFO') self.environ = environ self.version = version self.websocket_closed = False self._buf = "" self._msgs = collections.deque() self._sendlock = semaphore.Semaphore()
def setUp(self): super(SchedulerTest, self).setUp() # This Timeout object is needed to raise an exception if the test took # longer than a configured number of seconds. self.timeout = timeout.Timeout(seconds=15) # Synchronization primitives to control when a scheduled invoked # method is allowed to enter the method and exit from it to perform # all needed checks. self.target_mtd_started = event.Event() self.target_mtd_finished = event.Event() self.target_mtd_lock = semaphore.Semaphore(0) self.scheduler = default_scheduler.DefaultScheduler(1, 1, 100) self.scheduler.start() self.addCleanup(self.scheduler.stop, True) self.addCleanup(self.timeout.cancel)
def __init__(self, user_id=None, tenant_id=None, auth_token=None, service_catalog=None, username=None, tenant_name=None, roles=None, is_admin=None, remote_semaphore=None, auth_uri=None, resource_uuid=None, current_instance_info=None, request_id=None, overwrite=True, **kwargs): if kwargs: LOG.warning(_LW('Arguments dropped when creating context: ' '{args}').format(args=kwargs)) super(Context, self).__init__(auth_token=auth_token, user=user_id, tenant=tenant_id, is_admin=is_admin, resource_uuid=resource_uuid, request_id=request_id) self.service_catalog = service_catalog self.username = username self.tenant_name = tenant_name self.remote_semaphore = remote_semaphore or semaphore.Semaphore( CONF.cluster_remote_threshold) self.roles = roles if auth_uri: self.auth_uri = auth_uri else: self.auth_uri = _get_auth_uri() if overwrite or not hasattr(context._request_store, 'context'): self.update_store() if current_instance_info is not None: self.current_instance_info = current_instance_info else: self.current_instance_info = InstanceInfo()
class RpcConnection(object): CONN_LOCK = semaphore.Semaphore() def __init__(self, confirm_delivery=False): self.confirm_delivery = confirm_delivery self.connect() def connect(self): host = FLAGS.get('rabbit_host') port = FLAGS.get('rabbit_port') LOG.info(_("Connecting to rabbit_host %s %d") % (host, port)) credentials = pika.PlainCredentials(FLAGS.get('rabbit_userid'), FLAGS.get('rabbit_password')) con_param = pika.ConnectionParameters( host=FLAGS.get('rabbit_host'), port=FLAGS.get('rabbit_port'), credentials=credentials, virtual_host=FLAGS.get('rabbit_virtual_host')) queue_args = {"x-ha-policy": "all"} msg = 'AMQP connection exception occurred %d time(s)... retrying.' max_retries = 5 for i in range(max_retries + 1): with self.CONN_LOCK: try: self.conn = pika.BlockingConnection(con_param) self.channel = self.conn.channel() self.channel.queue_declare(queue='metric_queue', durable=True, arguments=queue_args) if self.confirm_delivery: self.channel.confirm_delivery() return except AMQPConnectionError: if i < max_retries: LOG.warn(_(msg) % i) time.sleep(2 * i) else: raise
def __init__(self, *args, **kwargs): super(Policy, self).__init__(*args, **kwargs) hdlr = logging.StreamHandler() fmt_str = '%(asctime)s - %(levelname)s - %(name)s - %(message)s' hdlr.setFormatter(logging.Formatter(fmt_str)) self.logger.addHandler(hdlr) self.logger.propagate = False if self.CONF.enable_debugger: self.logger.setLevel(logging.DEBUG) self.nib = app_manager.lookup_service_brick("NIB") self.dpset = app_manager.lookup_service_brick("dpset") self.matches = {} #stored josndict self.queueref = {} #[port][queue] self.requestQ = [] #a request queue self.sem = semaphore.Semaphore(1) #TO protect self.requestQ
def __init__(self, host, conf=None): if conf: self.conf = conf else: self.conf = cfg.CONF self.router_info = {} if not self.conf.interface_driver: LOG.error(_('An interface driver must be specified')) sys.exit(1) try: self.driver = importutils.import_object(self.conf.interface_driver, self.conf) except: LOG.exception( _("Error importing interface driver '%s'" % self.conf.interface_driver)) sys.exit(1) self.plugin_rpc = L3PluginApi(topics.PLUGIN, host) self.fullsync = True self.sync_sem = semaphore.Semaphore(1) if self.conf.use_namespaces: self._destroy_all_router_namespaces() super(L3NATAgent, self).__init__(host=self.conf.host)
def empty(self): while self.free_items: self.get().close() # Force a new connection pool to be created. # Note that this was added due to failing unit test cases. The issue # is the above "while loop" gets all the cached connections from the # pool and closes them, but never returns them to the pool, a pool # leak. The unit tests hang waiting for an item to be returned to the # pool. The unit tests get here via the tearDown() method. In the run # time code, it gets here via cleanup() and only appears in service.py # just before doing a sys.exit(), so cleanup() only happens once and # the leakage is not a problem. self.connection_cls.pool = None _pool_create_sem = semaphore.Semaphore() def get_connection_pool(conf, connection_cls): with _pool_create_sem: # Make sure only one thread tries to create the connection pool. if not connection_cls.pool: connection_cls.pool = Pool(conf, connection_cls) return connection_cls.pool class ConnectionContext(rpc_common.Connection): """The class that is actually returned to the create_connection() caller. This is essentially a wrapper around Connection that supports 'with'. It can also return a new Connection, or one from a pool.
self.conf = conf kwargs.setdefault("max_size", self.conf.rpc_conn_pool_size) kwargs.setdefault("order_as_stack", True) super(Pool, self).__init__(*args, **kwargs) # TODO(comstud): Timeout connections not used in a while def create(self): LOG.debug('Pool creating new connection') return self.connection_cls(self.conf) def empty(self): while self.free_items: self.get().close() _pool_create_sem = semaphore.Semaphore() def get_connection_pool(conf, connection_cls): with _pool_create_sem: # Make sure only one thread tries to create the connection pool. if not connection_cls.pool: connection_cls.pool = Pool(conf, connection_cls) return connection_cls.pool class ConnectionContext(rpc_common.Connection): """The class that is actually returned to the caller of create_connection(). This is essentially a wrapper around Connection that supports 'with'. It can also return a new Connection, or one from a pool. The function will also catch
def __init__(self): self._periodics_worker = None self._zeroconf = None self._shutting_down = semaphore.Semaphore()
def lock(name, lock_file_prefix=None, external=False, lock_path=None): """Context based lock This function yields a `semaphore.Semaphore` instance unless external is True, in which case, it'll yield an InterProcessLock instance. :param lock_file_prefix: The lock_file_prefix argument is used to provide lock files on disk with a meaningful prefix. :param external: The external keyword argument denotes whether this lock should work across multiple processes. This means that if two different workers both run a a method decorated with @synchronized('mylock', external=True), only one of them will execute at a time. :param lock_path: The lock_path keyword argument is used to specify a special location for external lock files to live. If nothing is set, then CONF.lock_path is used as a default. """ # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, semaphore.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) # NOTE(mikal): I know this looks odd if not hasattr(local.strong_store, 'locks_held'): local.strong_store.locks_held = [] local.strong_store.locks_held.append(name) try: if external and not CONF.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s"'), {'lock': name}) # We need a copy of lock_path because it is non-local local_lock_path = lock_path or CONF.lock_path if not local_lock_path: raise cfg.RequiredOptError('lock_path') if not os.path.exists(local_lock_path): fileutils.ensure_tree(local_lock_path) LOG.info(_('Created lock path: %s'), local_lock_path) def add_prefix(name, prefix): if not prefix: return name sep = '' if prefix.endswith('-') else '-' return '%s%s%s' % (prefix, sep, name) # NOTE(mikal): the lock name cannot contain directory # separators lock_file_name = add_prefix(name.replace(os.sep, '_'), lock_file_prefix) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock as lock: LOG.debug(_('Got file lock "%(lock)s" at %(path)s'), { 'lock': name, 'path': lock_file_path }) yield lock finally: LOG.debug(_('Released file lock "%(lock)s" at %(path)s'), { 'lock': name, 'path': lock_file_path }) else: yield sem finally: local.strong_store.locks_held.remove(name)