def process_request(self, request): roles = request.headers.get('X_ROLE', '').split(',') LOG.debug(_("Processing auth request with roles: %s") % roles) tenant_id = request.headers.get('X-Tenant-Id', None) LOG.debug(_("Processing auth request with tenant_id: %s") % tenant_id) for provider in self.auth_providers: provider.authorize(request, tenant_id, roles)
def start_mysql(self, update_db=False): LOG.info(_("Starting mysql...")) # This is the site of all the trouble in the restart tests. # Essentially what happens is that mysql start fails, but does not # die. It is then impossible to kill the original, so self._enable_mysql_on_boot() try: utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "start") except exception.ProcessExecutionError: # it seems mysql (percona, at least) might come back with [Fail] # but actually come up ok. we're looking into the timing issue on # parallel, but for now, we'd like to give it one more chance to # come up. so regardless of the execute_with_timeout() respose, # we'll assume mysql comes up and check it's status for a while. pass if not self.status.wait_for_real_status_to_change_to( rd_models.ServiceStatuses.RUNNING, self.state_change_wait_time, update_db): LOG.error(_("Start up of MySQL failed!")) # If it won't start, but won't die either, kill it by hand so we # don't let a rouge process wander around. try: utils.execute_with_timeout("sudo", "pkill", "-9", "mysql") except exception.ProcessExecutionError, p: LOG.error("Error killing stalled mysql start command.") LOG.error(p) # There's nothing more we can do... self.status.end_install_or_restart() raise RuntimeError("Could not start MySQL!")
def _multi_send(method, context, topic, msg, timeout=None, envelope=False, _msg_id=None): """ Wraps the sending of messages, dispatches to the matchmaker and sends message to all relevant hosts. """ conf = CONF LOG.debug(_("%(msg)s") % {'msg': ' '.join(map(pformat, (topic, msg)))}) queues = _get_matchmaker().queues(topic) LOG.debug(_("Sending message(s) to: %s"), queues) # Don't stack if we have no matchmaker results if len(queues) == 0: LOG.warn(_("No matchmaker results. Not casting.")) # While not strictly a timeout, callers know how to handle # this exception and a timeout isn't too big a lie. raise rpc_common.Timeout(_("No match from matchmaker.")) # This supports brokerless fanout (addresses > 1) for queue in queues: (_topic, ip_addr) = queue _addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port) if method.__name__ == '_cast': eventlet.spawn_n(method, _addr, context, _topic, msg, timeout, envelope, _msg_id) return return method(_addr, context, _topic, msg, timeout, envelope)
def index(self, req, tenant_id): """Return all storage devices.""" LOG.info(_("req : '%s'\n\n") % req) LOG.info(_("Indexing storage info for tenant '%s'") % tenant_id) context = req.environ[wsgi.CONTEXT_KEY] storages = models.StorageDevices.load(context) return wsgi.Result(views.StoragesView(storages).data(), 200)
def index(self, req, tenant_id, detailed=False): """Return all hosts.""" LOG.info(_("req : '%s'\n\n") % req) LOG.info(_("Indexing a host for tenant '%s'") % tenant_id) context = req.environ[wsgi.CONTEXT_KEY] hosts = models.SimpleHost.load_all(context) return wsgi.Result(views.HostsView(hosts).data(), 200)
def _get_response(self, ctx, proxy, topic, data): """Process a curried message and cast the result to topic.""" LOG.debug(_("Running func with context: %s"), ctx.to_dict()) data.setdefault('version', None) data.setdefault('args', {}) try: result = proxy.dispatch(ctx, data['version'], data['method'], data.get('namespace'), **data['args']) return ConsumerBase.normalize_reply(result, ctx.replies) except greenlet.GreenletExit: # ignore these since they are just from shutdowns pass except rpc_common.ClientException as e: LOG.debug( _("Expected exception during message handling (%s)") % e._exc_info[1]) return { 'exc': rpc_common.serialize_remote_exception(e._exc_info, log_failure=False) } except Exception: LOG.error(_("Exception during message handling")) return { 'exc': rpc_common.serialize_remote_exception(sys.exc_info()) }
def _error_callback(exc): if isinstance(exc, socket.timeout): LOG.exception(_("Timed out waiting for RPC response: %s") % str(exc)) raise rpc_common.Timeout() else: LOG.exception(_("Failed to consume message from queue: %s") % str(exc)) info["do_consume"] = True
def wait(self): """Loop waiting on children to die and respawning as necessary""" LOG.debug(_('Full set of CONF:')) CONF.log_opt_values(LOG, std_logging.DEBUG) while self.running: wrap = self._wait_child() if not wrap: continue while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap) if self.sigcaught: signame = {signal.SIGTERM: 'SIGTERM', signal.SIGINT: 'SIGINT'}[self.sigcaught] LOG.info(_('Caught %s, stopping children'), signame) for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def delete(self): if self.is_building: raise exception.UnprocessableEntity("Instance %s is not ready." % self.id) LOG.debug(_(" ... deleting compute id = %s") % self.db_info.compute_instance_id) LOG.debug(_(" ... setting status to DELETING.")) self.update_db(task_status=InstanceTasks.DELETING) task_api.API(self.context).delete_instance(self.id)
def _connect(self, params): """Connect to rabbit. Re-establish any queues that may have been declared before if we are reconnecting. Exceptions should be handled by the caller. """ if self.connection: LOG.info(_("Reconnecting to AMQP server on " "%(hostname)s:%(port)d") % params) try: self.connection.close() except self.connection_errors: pass # Setting this in case the next statement fails, though # it shouldn't be doing any network operations, yet. self.connection = None self.connection = kombu.connection.BrokerConnection(**params) self.connection_errors = self.connection.connection_errors if self.memory_transport: # Kludge to speed up tests. self.connection.transport.polling_interval = 0.0 self.consumer_num = itertools.count(1) self.connection.connect() self.channel = self.connection.channel() # work around 'memory' transport bug in 1.1.3 if self.memory_transport: self.channel._new_queue("ae.undeliver") for consumer in self.consumers: consumer.reconnect(self.channel) LOG.info(_("Connected to AMQP server on %(hostname)s:%(port)d") % params)
def reconnect(self): """Handles reconnecting and re-establishing sessions and queues""" if self.connection.opened(): try: self.connection.close() except qpid.messaging.exceptions.ConnectionError: pass attempt = 0 delay = 1 while True: broker = self.brokers[attempt % len(self.brokers)] attempt += 1 try: self.connection_create(broker) self.connection.open() except qpid.messaging.exceptions.ConnectionError, e: msg_dict = dict(e=e, delay=delay) msg = _("Unable to connect to AMQP server: %(e)s. " "Sleeping %(delay)s seconds") % msg_dict LOG.error(msg) time.sleep(delay) delay = min(2 * delay, 60) else: LOG.info(_('Connected to AMQP server on %s'), broker) break
def wait(self): """Loop waiting on children to die and respawning as necessary""" LOG.debug(_('Full set of CONF:')) CONF.log_opt_values(LOG, std_logging.DEBUG) while self.running: wrap = self._wait_child() if not wrap: continue while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap) if self.sigcaught: signame = { signal.SIGTERM: 'SIGTERM', signal.SIGINT: 'SIGINT' }[self.sigcaught] LOG.info(_('Caught %s, stopping children'), signame) for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def wait(self): """Loop waiting on children to die and respawning as necessary""" LOG.debug(_('Full set of CONF:')) CONF.log_opt_values(LOG, std_logging.DEBUG) while self.running: wrap = self._wait_child() if not wrap: # Yield to other threads if no children have exited # Sleep for a short time to avoid excessive CPU usage # (see bug #1095346) eventlet.greenthread.sleep(.01) continue while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap) if self.sigcaught: signame = {signal.SIGTERM: 'SIGTERM', signal.SIGINT: 'SIGINT'}[self.sigcaught] LOG.info(_('Caught %s, stopping children'), signame) for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def create_consumer(self, topic, proxy, fanout=False): # Only consume on the base topic name. topic = topic.split('.', 1)[0] LOG.info(_("Create Consumer for topic (%(topic)s)") % {'topic': topic}) # Subscription scenarios if fanout: subscribe = ('', fanout)[type(fanout) == str] sock_type = zmq.SUB topic = 'fanout~' + topic else: sock_type = zmq.PULL subscribe = None # Receive messages from (local) proxy inaddr = "ipc://%s/zmq_topic_%s" % \ (self.conf.rpc_zmq_ipc_dir, topic) LOG.debug(_("Consumer is a zmq.%s"), ['PULL', 'SUB'][sock_type == zmq.SUB]) self.reactor.register(proxy, inaddr, sock_type, subscribe=subscribe, in_bind=False)
def start_mysql(self, update_db=False): LOG.info(_("Starting mysql...")) # This is the site of all the trouble in the restart tests. # Essentially what happens is thaty mysql start fails, but does not # die. It is then impossible to kill the original, so self._enable_mysql_on_boot() try: utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "start") except ProcessExecutionError: # it seems mysql (percona, at least) might come back with [Fail] # but actually come up ok. we're looking into the timing issue on # parallel, but for now, we'd like to give it one more chance to # come up. so regardless of the execute_with_timeout() respose, # we'll assume mysql comes up and check it's status for a while. pass if not self.status.wait_for_real_status_to_change_to( rd_models.ServiceStatuses.RUNNING, self.state_change_wait_time, update_db): LOG.error(_("Start up of MySQL failed!")) # If it won't start, but won't die either, kill it by hand so we # don't let a rouge process wander around. try: utils.execute_with_timeout("sudo", "pkill", "-9", "mysql") except ProcessExecutionError, p: LOG.error("Error killing stalled mysql start command.") LOG.error(p) # There's nothing more we can do... self.status.end_install_or_restart() raise RuntimeError("Could not start MySQL!")
def register_opts(conf): """Registration of options for this driver.""" #NOTE(ewindisch): ZMQ_CTX and matchmaker # are initialized here as this is as good # an initialization method as any. # We memoize through these globals global ZMQ_CTX global matchmaker global FLAGS if not FLAGS: conf.register_opts(zmq_opts) FLAGS = conf # Don't re-set, if this method is called twice. if not ZMQ_CTX: ZMQ_CTX = zmq.Context(conf.rpc_zmq_contexts) if not matchmaker: # rpc_zmq_matchmaker should be set to a 'module.Class' mm_path = conf.rpc_zmq_matchmaker.split('.') mm_module = '.'.join(mm_path[:-1]) mm_class = mm_path[-1] # Only initialize a class. if mm_path[-1][0] not in string.ascii_uppercase: LOG.error( _("Matchmaker could not be loaded.\n" "rpc_zmq_matchmaker is not a class.")) raise RPCException(_("Error loading Matchmaker.")) mm_impl = importutils.import_module(mm_module) mm_constructor = getattr(mm_impl, mm_class) matchmaker = mm_constructor()
def action(self, req, body, tenant_id, id): LOG.info("req : '%s'\n\n" % req) LOG.info("Comitting an ACTION again instance %s for tenant '%s'" % (id, tenant_id)) if not body: raise exception.BadRequest(_("Invalid request body.")) context = req.environ[wsgi.CONTEXT_KEY] instance = models.Instance.load(context, id) _actions = { 'restart': self._action_restart, 'resize': self._action_resize, 'reset_password': self._action_reset_password } selected_action = None for key in body: if key in _actions: if selected_action is not None: msg = _("Only one action can be specified per request.") raise exception.BadRequest(msg) selected_action = _actions[key] else: msg = _("Invalid instance action: %s") % key raise exception.BadRequest(msg) if selected_action: return selected_action(instance, body) else: raise exception.BadRequest(_("Invalid request body."))
def publisher(waiter): LOG.info(_("Creating proxy for topic: %s"), topic) try: # The topic is received over the network, # don't trust this input. if self.badchars.search(topic) is not None: emsg = _("Topic contained dangerous characters.") LOG.warn(emsg) raise RPCException(emsg) out_sock = ZmqSocket("ipc://%s/zmq_topic_%s" % (ipc_dir, topic), sock_type, bind=True) except RPCException: waiter.send_exception(*sys.exc_info()) return self.topic_proxy[topic] = eventlet.queue.LightQueue( CONF.rpc_zmq_topic_backlog) self.sockets.append(out_sock) # It takes some time for a pub socket to open, # before we can have any faith in doing a send() to it. if sock_type == zmq.PUB: eventlet.sleep(.5) waiter.send(True) while(True): data = self.topic_proxy[topic].get() out_sock.send(data) LOG.debug(_("ROUTER RELAY-OUT SUCCEEDED %(data)s") % {'data': data})
def action(self, req, body, tenant_id, id): LOG.info("req : '%s'\n\n" % req) LOG.info("Committing an ACTION against instance %s for tenant '%s'" % (id, tenant_id)) if not body: raise exception.BadRequest(_("Invalid request body.")) context = req.environ[wsgi.CONTEXT_KEY] instance = models.MgmtInstance.load(context=context, id=id) _actions = { 'stop': self._action_stop, 'reboot': self._action_reboot, 'migrate': self._action_migrate, 'reset-task-status': self._action_reset_task_status } selected_action = None for key in body: if key in _actions: if selected_action is not None: msg = _("Only one action can be specified per request.") raise exception.BadRequest(msg) selected_action = _actions[key] else: msg = _("Invalid instance action: %s") % key raise exception.BadRequest(msg) if selected_action: return selected_action(context, instance, body) else: raise exception.BadRequest(_("Invalid request body."))
def consume_in_thread(self): """Runs the ZmqProxy service""" ipc_dir = CONF.rpc_zmq_ipc_dir consume_in = "tcp://%s:%s" % \ (CONF.rpc_zmq_bind_address, CONF.rpc_zmq_port) consumption_proxy = InternalContext(None) if not os.path.isdir(ipc_dir): try: utils.execute('mkdir', '-p', ipc_dir, run_as_root=True) utils.execute('chown', "%s:%s" % (os.getuid(), os.getgid()), ipc_dir, run_as_root=True) utils.execute('chmod', '750', ipc_dir, run_as_root=True) except utils.ProcessExecutionError: with excutils.save_and_reraise_exception(): LOG.error(_("Could not create IPC directory %s") % (ipc_dir, )) try: self.register(consumption_proxy, consume_in, zmq.PULL, out_bind=True) except zmq.ZMQError: with excutils.save_and_reraise_exception(): LOG.error(_("Could not create ZeroMQ receiver daemon. " "Socket may already be in use.")) super(ZmqProxy, self).consume_in_thread()
def consume(self, sock): #TODO(ewindisch): use zero-copy (i.e. references, not copying) data = sock.recv() LOG.debug(_("CONSUMER RECEIVED DATA: %s"), data) if sock in self.mapping: LOG.debug(_("ROUTER RELAY-OUT %(data)s") % {'data': data}) self.mapping[sock].send(data) return proxy = self.proxies[sock] if data[2] == 'cast': # Legacy protocol packenv = data[3] ctx, msg = _deserialize(packenv) request = rpc_common.deserialize_msg(msg) ctx = RpcContext.unmarshal(ctx) elif data[2] == 'impl_zmq_v2': packenv = data[4:] msg = unflatten_envelope(packenv) request = rpc_common.deserialize_msg(msg) # Unmarshal only after verifying the message. ctx = RpcContext.unmarshal(data[3]) else: LOG.error(_("ZMQ Envelope version unsupported or unknown.")) return self.pool.spawn_n(self.process, proxy, ctx, request)
def consume(self, sock): #TODO(ewindisch): use zero-copy (i.e. references, not copying) data = sock.recv() LOG.debug(_("CONSUMER RECEIVED DATA: %s"), data) if sock in self.mapping: LOG.debug(_("ROUTER RELAY-OUT %(data)s") % { 'data': data}) self.mapping[sock].send(data) return proxy = self.proxies[sock] if data[2] == 'cast': # Legacy protocol packenv = data[3] ctx, msg = _deserialize(packenv) request = rpc_common.deserialize_msg(msg) ctx = RpcContext.unmarshal(ctx) elif data[2] == 'impl_zmq_v2': packenv = data[4:] msg = unflatten_envelope(packenv) request = rpc_common.deserialize_msg(msg) # Unmarshal only after verifying the message. ctx = RpcContext.unmarshal(data[3]) else: LOG.error(_("ZMQ Envelope version unsupported or unknown.")) return self.pool.spawn_n(self.process, proxy, ctx, request)
def update(self, req, body, tenant_id, instance_id): """Change the password of one or more users.""" LOG.info(_("Updating user passwords for instance '%s'") % instance_id) LOG.info(_("req : '%s'\n\n") % req) context = req.environ[wsgi.CONTEXT_KEY] self.validate(body) users = body['users'] model_users = [] for user in users: try: mu = guest_models.MySQLUser() mu.name = user['name'] mu.host = user.get('host') mu.password = user['password'] found_user = models.User.load(context, instance_id, mu.name, mu.host) if not found_user: user_and_host = mu.name if mu.host: user_and_host += '@' + mu.host raise exception.UserNotFound(uuid=user_and_host) model_users.append(mu) except (ValueError, AttributeError) as e: raise exception.BadRequest(msg=str(e)) models.User.change_password(context, instance_id, model_users) return wsgi.Result(None, 202)
def create_consumer(self, topic, proxy, fanout=False): # Register with matchmaker. _get_matchmaker().register(topic, CONF.rpc_zmq_host) # Subscription scenarios if fanout: sock_type = zmq.SUB subscribe = ('', fanout)[type(fanout) == str] topic = 'fanout~' + topic.split('.', 1)[0] else: sock_type = zmq.PULL subscribe = None topic = '.'.join((topic.split('.', 1)[0], CONF.rpc_zmq_host)) if topic in self.topics: LOG.info(_("Skipping topic registration. Already registered.")) return # Receive messages from (local) proxy inaddr = "ipc://%s/zmq_topic_%s" % \ (CONF.rpc_zmq_ipc_dir, topic) LOG.debug(_("Consumer is a zmq.%s"), ['PULL', 'SUB'][sock_type == zmq.SUB]) self.reactor.register(proxy, inaddr, sock_type, subscribe=subscribe, in_bind=False) self.topics.append(topic)
def _wait_child(self): try: # Don't block if no child processes have exited pid, status = os.waitpid(0, os.WNOHANG) if not pid: return None except OSError as exc: if exc.errno not in (errno.EINTR, errno.ECHILD): raise return None if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) LOG.info(_('Child %(pid)d killed by signal %(sig)d'), dict(pid=pid, sig=sig)) else: code = os.WEXITSTATUS(status) LOG.info(_('Child %(pid)s exited with status %(code)d'), dict(pid=pid, code=code)) if pid not in self.children: LOG.warning(_('pid %d not in child list'), pid) return None wrap = self.children.pop(pid) wrap.children.remove(pid) return wrap
def __init__(self, addr, zmq_type, bind=True, subscribe=None): self.sock = ZMQ_CTX.socket(zmq_type) self.addr = addr self.type = zmq_type self.subscriptions = [] # Support failures on sending/receiving on wrong socket type. self.can_recv = zmq_type in (zmq.PULL, zmq.SUB) self.can_send = zmq_type in (zmq.PUSH, zmq.PUB) self.can_sub = zmq_type in (zmq.SUB, ) # Support list, str, & None for subscribe arg (cast to list) do_sub = { list: subscribe, str: [subscribe], type(None): [] }[type(subscribe)] for f in do_sub: self.subscribe(f) str_data = {'addr': addr, 'type': self.socket_s(), 'subscribe': subscribe, 'bind': bind} LOG.debug(_("Connecting to %(addr)s with %(type)s"), str_data) LOG.debug(_("-> Subscribed to %(subscribe)s"), str_data) LOG.debug(_("-> bind: %(bind)s"), str_data) try: if bind: self.sock.bind(addr) else: self.sock.connect(addr) except Exception: raise RPCException(_("Could not open socket."))
def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" for task_name, task in self._periodic_tasks: full_task_name = '.'.join([self.__class__.__name__, task_name]) ticks_to_skip = self._ticks_to_skip[task_name] if ticks_to_skip > 0: LOG.debug( _("Skipping %(full_task_name)s, %(ticks_to_skip)s" " ticks left until next run"), dict(full_task_name=full_task_name, ticks_to_skip=ticks_to_skip)) self._ticks_to_skip[task_name] -= 1 continue self._ticks_to_skip[task_name] = task._ticks_between_runs LOG.debug(_("Running periodic task %(full_task_name)s"), dict(full_task_name=full_task_name)) try: task(self, context) except Exception as e: if raise_on_error: raise LOG.exception(_("Error during %(full_task_name)s:" " %(e)s"), dict(e=e, full_task_name=full_task_name))
def register(self, proxy, in_addr, zmq_type_in, out_addr=None, zmq_type_out=None, in_bind=True, out_bind=True, subscribe=None): LOG.info(_("Registering reactor")) if zmq_type_in not in (zmq.PULL, zmq.SUB): raise RPCException("Bad input socktype") # Items push in. inq = ZmqSocket(in_addr, zmq_type_in, bind=in_bind, subscribe=subscribe) self.proxies[inq] = proxy self.sockets.append(inq) LOG.info(_("In reactor registered")) if not out_addr: return if zmq_type_out not in (zmq.PUSH, zmq.PUB): raise RPCException("Bad output socktype") # Items push out. outq = ZmqSocket(out_addr, zmq_type_out, bind=out_bind) self.mapping[inq] = outq self.mapping[outq] = inq self.sockets.append(outq) LOG.info(_("Out reactor registered"))
def _wait_child(self): try: # Don't block if no child processes have exited pid, status = os.waitpid(0, os.WNOHANG) if not pid: return None except OSError as exc: if exc.errno not in (errno.EINTR, errno.ECHILD): raise return None if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) LOG.info(_('Child %(pid)d killed by signal %(sig)d'), dict(pid=pid, sig=sig)) else: code = os.WEXITSTATUS(status) LOG.info(_('Child %(pid)s exited with status %(code)d'), dict(pid=pid, code=code)) if pid not in self.children: LOG.warning(_('pid %d not in child list'), pid) return None wrap = self.children.pop(pid) wrap.children.remove(pid) return wrap
def create_consumer(self, topic, proxy, fanout=False): # Only consume on the base topic name. topic = topic.split('.', 1)[0] LOG.info(_("Create Consumer for topic (%(topic)s)") % {'topic': topic}) # Subscription scenarios if fanout: subscribe = ('', fanout)[type(fanout) == str] sock_type = zmq.SUB topic = 'fanout~' + topic else: sock_type = zmq.PULL subscribe = None # Receive messages from (local) proxy inaddr = "ipc://%s/zmq_topic_%s" % \ (CONF.rpc_zmq_ipc_dir, topic) LOG.debug(_("Consumer is a zmq.%s"), ['PULL', 'SUB'][sock_type == zmq.SUB]) self.reactor.register(proxy, inaddr, sock_type, subscribe=subscribe, in_bind=False)
def _load_servers_status(load_instance, context, db_items, find_server): ret = [] for db in db_items: server = None try: #TODO(tim.simpson): Delete when we get notifications working! if InstanceTasks.BUILDING == db.task_status: db.server_status = "BUILD" else: try: server = find_server(db.id, db.compute_instance_id) db.server_status = server.status except exception.ComputeInstanceNotFound: db.server_status = "SHUTDOWN" # Fake it... #TODO(tim.simpson): End of hack. #volumes = find_volumes(server.id) status = InstanceServiceStatus.find_by(instance_id=db.id) LOG.info( _("Server api_status(%s)") % (status.status.api_status)) if not status.status: # This should never happen. LOG.error( _("Server status could not be read for " "instance id(%s)") % (db.id)) continue except exception.ModelNotFoundError: LOG.error( _("Server status could not be read for " "instance id(%s)") % (db.id)) continue ret.append(load_instance(context, db, status)) return ret
def _multi_send(method, context, topic, msg, timeout=None): """ Wraps the sending of messages, dispatches to the matchmaker and sends message to all relevant hosts. """ conf = CONF LOG.debug(_("%(msg)s") % {'msg': ' '.join(map(pformat, (topic, msg)))}) queues = matchmaker.queues(topic) LOG.debug(_("Sending message(s) to: %s"), queues) # Don't stack if we have no matchmaker results if len(queues) == 0: LOG.warn(_("No matchmaker results. Not casting.")) # While not strictly a timeout, callers know how to handle # this exception and a timeout isn't too big a lie. raise rpc_common.Timeout, "No match from matchmaker." # This supports brokerless fanout (addresses > 1) for queue in queues: (_topic, ip_addr) = queue _addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port) if method.__name__ == '_cast': eventlet.spawn_n(method, _addr, context, _topic, _topic, msg, timeout) return return method(_addr, context, _topic, _topic, msg, timeout)
def index(self, req, tenant_id, detailed=False): """Return all hosts.""" LOG.info(_("req : '%s'\n\n") % req) LOG.info(_("Indexing a host for tenant '%s'") % tenant_id) context = req.environ[wsgi.CONTEXT_KEY] hosts = models.SimpleHost.load_all(context) return wsgi.Result(views.HostsView(hosts).data(), 200)
def register_opts(conf): """Registration of options for this driver.""" #NOTE(ewindisch): ZMQ_CTX and matchmaker # are initialized here as this is as good # an initialization method as any. # We memoize through these globals global ZMQ_CTX global matchmaker global CONF if not CONF: conf.register_opts(zmq_opts) CONF = conf # Don't re-set, if this method is called twice. if not ZMQ_CTX: ZMQ_CTX = zmq.Context(conf.rpc_zmq_contexts) if not matchmaker: # rpc_zmq_matchmaker should be set to a 'module.Class' mm_path = conf.rpc_zmq_matchmaker.split('.') mm_module = '.'.join(mm_path[:-1]) mm_class = mm_path[-1] # Only initialize a class. if mm_path[-1][0] not in string.ascii_uppercase: LOG.error(_("Matchmaker could not be loaded.\n" "rpc_zmq_matchmaker is not a class.")) raise RPCException(_("Error loading Matchmaker.")) mm_impl = importutils.import_module(mm_module) mm_constructor = getattr(mm_impl, mm_class) matchmaker = mm_constructor()
def _connect(self, params): """Connect to rabbit. Re-establish any queues that may have been declared before if we are reconnecting. Exceptions should be handled by the caller. """ if self.connection: LOG.info( _("Reconnecting to AMQP server on " "%(hostname)s:%(port)d") % params) try: self.connection.close() except self.connection_errors: pass # Setting this in case the next statement fails, though # it shouldn't be doing any network operations, yet. self.connection = None self.connection = kombu.connection.BrokerConnection(**params) self.connection_errors = self.connection.connection_errors if self.memory_transport: # Kludge to speed up tests. self.connection.transport.polling_interval = 0.0 self.consumer_num = itertools.count(1) self.connection.connect() self.channel = self.connection.channel() # work around 'memory' transport bug in 1.1.3 if self.memory_transport: self.channel._new_queue('ae.undeliver') for consumer in self.consumers: consumer.reconnect(self.channel) LOG.info( _('Connected to AMQP server on %(hostname)s:%(port)d') % params)
def _parse_check(rule): """ Parse a single base check rule into an appropriate Check object. """ # Handle the special checks if rule == '!': return FalseCheck() elif rule == '@': return TrueCheck() try: kind, match = rule.split(':', 1) except Exception: LOG.exception(_("Failed to understand rule %(rule)s") % locals()) # If the rule is invalid, we'll fail closed return FalseCheck() # Find what implements the check if kind in _checks: return _checks[kind](kind, match) elif None in _checks: return _checks[None](kind, match) else: LOG.error(_("No handler for matches of kind %s") % kind) return FalseCheck()
def _action_resize(self, instance, body): """ Handles 2 cases 1. resize volume body only contains {volume: {size: x}} 2. resize instance body only contains {flavorRef: http.../2} If the body has both we will throw back an error. """ options = { 'volume': self._action_resize_volume, 'flavorRef': self._action_resize_flavor } selected_option = None args = None for key in options: if key in body['resize']: if selected_option is not None: msg = _("Not allowed to resize volume and flavor at the " "same time.") raise exception.BadRequest(msg) selected_option = options[key] args = body['resize'][key] if selected_option: return selected_option(instance, args) else: raise exception.BadRequest(_("Missing resize arguments."))
def reconnect(self): """Handles reconnecting and re-establishing sessions and queues""" if self.connection.opened(): try: self.connection.close() except qpid.messaging.exceptions.ConnectionError: pass attempt = 0 delay = 1 while True: broker = self.brokers[attempt % len(self.brokers)] attempt += 1 try: self.connection_create(broker) self.connection.open() except qpid.messaging.exceptions.ConnectionError, e: msg_dict = dict(e=e, delay=delay) msg = _("Unable to connect to AMQP server: %(e)s. " "Sleeping %(delay)s seconds") % msg_dict LOG.error(msg) time.sleep(delay) delay = min(2 * delay, 60) else: LOG.info(_('Connected to AMQP server on %s'), broker) break
def is_enabled(): cert_file = CONF.ssl.cert_file key_file = CONF.ssl.key_file ca_file = CONF.ssl.ca_file use_ssl = cert_file or key_file if cert_file and not os.path.exists(cert_file): raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) if ca_file and not os.path.exists(ca_file): raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) if key_file and not os.path.exists(key_file): raise RuntimeError(_("Unable to find key_file : %s") % key_file) if use_ssl and (not cert_file or not key_file): raise RuntimeError( _( "When running server in SSL mode, you must " "specify both a cert_file and key_file " "option value in your configuration file" ) ) return use_ssl
def process_request(self, request): roles = request.headers.get('X_ROLE', '').split(',') LOG.debug(_("Processing auth request with roles: %s") % roles) tenant_id = request.headers.get('X-Tenant-Id', None) LOG.debug(_("Processing auth request with tenant_id: %s") % tenant_id) for provider in self.auth_providers: provider.authorize(request, tenant_id, roles)
def action(self, req, body, tenant_id, id): LOG.info("req : '%s'\n\n" % req) LOG.info("Committing an ACTION against instance %s for tenant '%s'" % (id, tenant_id)) if not body: raise exception.BadRequest(_("Invalid request body.")) context = req.environ[wsgi.CONTEXT_KEY] instance = models.MgmtInstance.load(context=context, id=id) _actions = { 'stop': self._action_stop, 'reboot': self._action_reboot, 'migrate': self._action_migrate, 'reset-task-status': self._action_reset_task_status } selected_action = None for key in body: if key in _actions: if selected_action is not None: msg = _("Only one action can be specified per request.") raise exception.BadRequest(msg) selected_action = _actions[key] else: msg = _("Invalid instance action: %s") % key raise exception.BadRequest(msg) if selected_action: return selected_action(context, instance, body) else: raise exception.BadRequest(_("Invalid request body."))
def __call__(self, message_data): """Consumer callback to call a method on a proxy object. Parses the message for validity and fires off a thread to call the proxy object method. Message data should be a dictionary with two keys: method: string representing the method to call args: dictionary of arg: value Example: {'method': 'echo', 'args': {'value': 42}} """ # It is important to clear the context here, because at this point # the previous context is stored in local.store.context if hasattr(local.store, 'context'): del local.store.context rpc_common._safe_log(LOG.debug, _('received %s'), message_data) self.msg_id_cache.check_duplicate_message(message_data) ctxt = unpack_context(self.conf, message_data) method = message_data.get('method') args = message_data.get('args', {}) version = message_data.get('version') namespace = message_data.get('namespace') if not method: LOG.warn(_('no method for message: %s') % message_data) ctxt.reply(_('No method for message: %s') % message_data, connection_pool=self.connection_pool) return self.pool.spawn_n(self._process_data, ctxt, version, method, namespace, args)
def consume_in_thread(self): """Runs the ZmqProxy service""" ipc_dir = CONF.rpc_zmq_ipc_dir consume_in = "tcp://%s:%s" % \ (CONF.rpc_zmq_bind_address, CONF.rpc_zmq_port) consumption_proxy = InternalContext(None) if not os.path.isdir(ipc_dir): try: utils.execute('mkdir', '-p', ipc_dir, run_as_root=True) utils.execute('chown', "%s:%s" % (os.getuid(), os.getgid()), ipc_dir, run_as_root=True) utils.execute('chmod', '750', ipc_dir, run_as_root=True) except utils.ProcessExecutionError: with excutils.save_and_reraise_exception(): LOG.error( _("Could not create IPC directory %s") % (ipc_dir, )) try: self.register(consumption_proxy, consume_in, zmq.PULL, out_bind=True) except zmq.ZMQError: with excutils.save_and_reraise_exception(): LOG.error( _("Could not create ZeroMQ receiver daemon. " "Socket may already be in use.")) super(ZmqProxy, self).consume_in_thread()
def _process_data(self, ctxt, version, method, namespace, args): """Process a message in a new thread. If the proxy object we have has a dispatch method (see rpc.dispatcher.RpcDispatcher), pass it the version, method, and args and let it dispatch as appropriate. If not, use the old behavior of magically calling the specified method on the proxy we have here. """ ctxt.update_store() try: rval = self.proxy.dispatch(ctxt, version, method, namespace, **args) # Check if the result was a generator if inspect.isgenerator(rval): for x in rval: ctxt.reply(x, None, connection_pool=self.connection_pool) else: ctxt.reply(rval, None, connection_pool=self.connection_pool) # This final None tells multicall that it is done. ctxt.reply(ending=True, connection_pool=self.connection_pool) except rpc_common.ClientException as e: LOG.debug(_('Expected exception during message handling (%s)') % e._exc_info[1]) ctxt.reply(None, e._exc_info, connection_pool=self.connection_pool, log_failure=False) except Exception: # sys.exc_info() is deleted by LOG.exception(). exc_info = sys.exc_info() LOG.error(_('Exception during message handling'), exc_info=exc_info) ctxt.reply(None, exc_info, connection_pool=self.connection_pool)
def create_consumer(self, topic, proxy, fanout=False): # Register with matchmaker. _get_matchmaker().register(topic, CONF.rpc_zmq_host) # Subscription scenarios if fanout: sock_type = zmq.SUB subscribe = ('', fanout)[type(fanout) == str] topic = 'fanout~' + topic.split('.', 1)[0] else: sock_type = zmq.PULL subscribe = None topic = '.'.join((topic.split('.', 1)[0], CONF.rpc_zmq_host)) if topic in self.topics: LOG.info(_("Skipping topic registration. Already registered.")) return # Receive messages from (local) proxy inaddr = "ipc://%s/zmq_topic_%s" % \ (CONF.rpc_zmq_ipc_dir, topic) LOG.debug(_("Consumer is a zmq.%s"), ['PULL', 'SUB'][sock_type == zmq.SUB]) self.reactor.register(proxy, inaddr, sock_type, subscribe=subscribe, in_bind=False) self.topics.append(topic)
def multicall(conf, context, topic, msg, timeout, connection_pool): """Make a call that returns multiple times.""" # TODO(pekowski): Remove all these comments in Havana. # For amqp_rpc_single_reply_queue = False, # Can't use 'with' for multicall, as it returns an iterator # that will continue to use the connection. When it's done, # connection.close() will get called which will put it back into # the pool # For amqp_rpc_single_reply_queue = True, # The 'with' statement is mandatory for closing the connection LOG.debug(_('Making synchronous call on %s ...'), topic) msg_id = uuid.uuid4().hex msg.update({'_msg_id': msg_id}) LOG.debug(_('MSG_ID is %s') % (msg_id)) _add_unique_id(msg) pack_context(msg, context) # TODO(pekowski): Remove this flag and the code under the if clause # in Havana. if not conf.amqp_rpc_single_reply_queue: conn = ConnectionContext(conf, connection_pool) wait_msg = MulticallWaiter(conf, conn, timeout) conn.declare_direct_consumer(msg_id, wait_msg) conn.topic_send(topic, rpc_common.serialize_msg(msg), timeout) else: with _reply_proxy_create_sem: if not connection_pool.reply_proxy: connection_pool.reply_proxy = ReplyProxy(conf, connection_pool) msg.update({'_reply_q': connection_pool.reply_proxy.get_reply_q()}) wait_msg = MulticallProxyWaiter(conf, msg_id, timeout, connection_pool) with ConnectionContext(conf, connection_pool) as conn: conn.topic_send(topic, rpc_common.serialize_msg(msg), timeout) return wait_msg
def _parse_check(rule): """ Parse a single base check rule into an appropriate Check object. """ # Handle the special checks if rule == '!': return FalseCheck() elif rule == '@': return TrueCheck() try: kind, match = rule.split(':', 1) except Exception: LOG.exception(_("Failed to understand rule %(rule)s") % locals()) # If the rule is invalid, we'll fail closed return FalseCheck() # Find what implements the check if kind in _checks: return _checks[kind](kind, match) elif None in _checks: return _checks[None](kind, match) else: LOG.error(_("No handler for matches of kind %s") % kind) return FalseCheck()
def inner(*args, **kwargs): # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, semaphore.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s" for method ' '"%(method)s"...'), {'lock': name, 'method': f.__name__}) if external and not CONF.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...'), {'lock': name, 'method': f.__name__}) cleanup_dir = False # We need a copy of lock_path because it is non-local local_lock_path = lock_path if not local_lock_path: local_lock_path = CONF.lock_path if not local_lock_path: cleanup_dir = True local_lock_path = tempfile.mkdtemp() if not os.path.exists(local_lock_path): cleanup_dir = True fileutils.ensure_tree(local_lock_path) # NOTE(mikal): the lock name cannot contain directory # separators safe_name = name.replace(os.sep, '_') lock_file_name = '%s%s' % (lock_file_prefix, safe_name) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock: LOG.debug(_('Got file lock "%(lock)s" at %(path)s ' 'for method "%(method)s"...'), {'lock': name, 'path': lock_file_path, 'method': f.__name__}) retval = f(*args, **kwargs) finally: # NOTE(vish): This removes the tempdir if we needed # to create one. This is used to cleanup # the locks left behind by unit tests. if cleanup_dir: shutil.rmtree(local_lock_path) else: retval = f(*args, **kwargs) return retval
def start(self): vcs_string = version.version_string_with_vcs() LOG.info(_('Starting %(topic)s node (version %(vcs_string)s)'), {'topic': self.topic, 'vcs_string': vcs_string}) self.conn = rpc.create_connection(new=True) LOG.debug(_("Creating Consumer connection for Service %s") % self.topic) # Share this same connection for these Consumers self.conn.create_consumer(self.topic, self, fanout=False) node_topic = '%s.%s' % (self.topic, self.host) self.conn.create_consumer(node_topic, self, fanout=False) self.conn.create_consumer(self.topic, self, fanout=True) # Consume from all consumers in a thread self.conn.consume_in_thread() if self.report_interval: pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) self.timers.append(pulse) if self.periodic_interval: periodic = utils.LoopingCall(self.periodic_tasks) periodic.start(interval=self.periodic_interval, now=False) self.timers.append(periodic)
def _action_resize(self, instance, body): """ Handles 2 cases 1. resize volume body only contains {volume: {size: x}} 2. resize instance body only contains {flavorRef: http.../2} If the body has both we will throw back an error. """ options = { 'volume': self._action_resize_volume, 'flavorRef': self._action_resize_flavor } selected_option = None args = None for key in options: if key in body['resize']: if selected_option is not None: msg = _("Not allowed to resize volume and flavor at the " "same time.") raise exception.BadRequest(msg) selected_option = options[key] args = body['resize'][key] if selected_option: return selected_option(instance, args) else: raise exception.BadRequest(_("Missing resize arguments."))
def create(self, req, body, tenant_id): # TODO(hub-cap): turn this into middleware LOG.info(_("Creating a database instance for tenant '%s'") % tenant_id) LOG.info(_("req : '%s'\n\n") % req) LOG.info(_("body : '%s'\n\n") % body) context = req.environ[wsgi.CONTEXT_KEY] # Set the service type to mysql if its not in the request service_type = (body['instance'].get('service_type') or CONF.service_type) service = models.ServiceImage.find_by(service_name=service_type) image_id = service['image_id'] name = body['instance']['name'] flavor_ref = body['instance']['flavorRef'] flavor_id = utils.get_id_from_href(flavor_ref) databases = populate_databases(body['instance'].get('databases', [])) users = populate_users(body['instance'].get('users', [])) if body['instance'].get('volume', None) is not None: try: volume_size = int(body['instance']['volume']['size']) except ValueError as e: raise exception.BadValue(msg=e) else: volume_size = None instance = models.Instance.create(context, name, flavor_id, image_id, databases, users, service_type, volume_size) view = views.InstanceDetailView(instance, req=req) return wsgi.Result(view.data(), 200)
def action(self, req, body, tenant_id, id): LOG.info("req : '%s'\n\n" % req) LOG.info("Comitting an ACTION again instance %s for tenant '%s'" % (id, tenant_id)) if not body: raise exception.BadRequest(_("Invalid request body.")) context = req.environ[wsgi.CONTEXT_KEY] instance = models.Instance.load(context, id) _actions = { 'restart': self._action_restart, 'resize': self._action_resize, 'reset_password': self._action_reset_password } selected_action = None for key in body: if key in _actions: if selected_action is not None: msg = _("Only one action can be specified per request.") raise exception.BadRequest(msg) selected_action = _actions[key] else: msg = _("Invalid instance action: %s") % key raise exception.BadRequest(msg) if selected_action: return selected_action(instance, body) else: raise exception.BadRequest(_("Invalid request body."))
def backups(self, req, tenant_id, id): """Return all backups for the specified instance.""" LOG.info(_("req : '%s'\n\n") % req) LOG.info(_("Indexing backups for instance '%s'") % id) backups = backup_model.list_for_instance(id) return wsgi.Result(backup_views.BackupViews(backups).data(), 200)
def _load_servers_status(load_instance, context, db_items, find_server): ret = [] for db in db_items: server = None try: #TODO(tim.simpson): Delete when we get notifications working! if InstanceTasks.BUILDING == db.task_status: db.server_status = "BUILD" else: try: server = find_server(db.id, db.compute_instance_id) db.server_status = server.status except exception.ComputeInstanceNotFound: db.server_status = "SHUTDOWN" # Fake it... #TODO(tim.simpson): End of hack. #volumes = find_volumes(server.id) status = InstanceServiceStatus.find_by(instance_id=db.id) LOG.info(_("Server api_status(%s)") % (status.status.api_status)) if not status.status: # This should never happen. LOG.error(_("Server status could not be read for " "instance id(%s)") % (db.id)) continue except exception.ModelNotFoundError: LOG.error(_("Server status could not be read for " "instance id(%s)") % (db.id)) continue ret.append(load_instance(context, db, status)) return ret
def _error_callback(exc): if isinstance(exc, qpid.messaging.exceptions.Empty): LOG.exception( _('Timed out waiting for RPC response: %s') % str(exc)) raise rpc_common.Timeout() else: LOG.exception( _('Failed to consume message from queue: %s') % str(exc))
def create(self, req, tenant_id, instance_id): """ Enable the root user for the db instance """ LOG.info(_("Enabling root for instance '%s'") % instance_id) LOG.info(_("req : '%s'\n\n") % req) context = req.environ[wsgi.CONTEXT_KEY] user_name = context.user root = models.Root.create(context, instance_id, user_name) return wsgi.Result(views.RootCreatedView(root).data(), 200)