def _wait_for_exit_or_signal(self): status = None signo = 0 LOG.debug(_('Full set of CONF:')) CONF.log_opt_values(LOG, std_logging.DEBUG) try: super(ServiceLauncher, self).wait() except SignalExit as exc: signame = {signal.SIGTERM: 'SIGTERM', signal.SIGINT: 'SIGINT', signal.SIGHUP: 'SIGHUP'}[exc.signo] LOG.info(_('Caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code finally: self.stop() if rpc: try: rpc.cleanup() except Exception: # We're shutting down, so it doesn't matter at this point. LOG.exception(_('Exception during rpc cleanup.')) return status, signo
def _connect(self, params): """Connect to rabbit. Re-establish any queues that may have been declared before if we are reconnecting. Exceptions should be handled by the caller. """ if self.connection: LOG.info(_("Reconnecting to AMQP server on " "%(hostname)s:%(port)d") % params) try: self.connection.release() except self.connection_errors: pass # Setting this in case the next statement fails, though # it shouldn't be doing any network operations, yet. self.connection = None self.connection = kombu.connection.BrokerConnection(**params) self.connection_errors = self.connection.connection_errors if self.memory_transport: # Kludge to speed up tests. self.connection.transport.polling_interval = 0.0 self.consumer_num = itertools.count(1) self.connection.connect() self.channel = self.connection.channel() # work around 'memory' transport bug in 1.1.3 if self.memory_transport: self.channel._new_queue('ae.undeliver') for consumer in self.consumers: consumer.reconnect(self.channel) LOG.info(_('Connected to AMQP server on %(hostname)s:%(port)d') % params)
def consume(self, sock): #TODO(ewindisch): use zero-copy (i.e. references, not copying) data = sock.recv() LOG.debug(_("CONSUMER RECEIVED DATA: %s"), data) proxy = self.proxies[sock] if data[2] == 'cast': # Legacy protocol packenv = data[3] ctx, msg = _deserialize(packenv) request = rpc_common.deserialize_msg(msg) ctx = RpcContext.unmarshal(ctx) elif data[2] == 'impl_zmq_v2': packenv = data[4:] msg = unflatten_envelope(packenv) request = rpc_common.deserialize_msg(msg) # Unmarshal only after verifying the message. ctx = RpcContext.unmarshal(data[3]) else: LOG.error(_("ZMQ Envelope version unsupported or unknown.")) return self.pool.spawn_n(self.process, proxy, ctx, request)
def consume_in_thread(self): """Runs the ZmqProxy service.""" ipc_dir = CONF.rpc_zmq_ipc_dir consume_in = "tcp://%s:%s" % \ (CONF.rpc_zmq_bind_address, CONF.rpc_zmq_port) consumption_proxy = InternalContext(None) try: os.makedirs(ipc_dir) except os.error: if not os.path.isdir(ipc_dir): with excutils.save_and_reraise_exception(): LOG.error(_("Required IPC directory does not exist at" " %s") % (ipc_dir, )) try: self.register(consumption_proxy, consume_in, zmq.PULL) except zmq.ZMQError: if os.access(ipc_dir, os.X_OK): with excutils.save_and_reraise_exception(): LOG.error(_("Permission denied to IPC directory at" " %s") % (ipc_dir, )) with excutils.save_and_reraise_exception(): LOG.error(_("Could not create ZeroMQ receiver daemon. " "Socket may already be in use.")) super(ZmqProxy, self).consume_in_thread()
def create_consumer(self, topic, proxy, fanout=False): # Register with matchmaker. _get_matchmaker().register(topic, CONF.rpc_zmq_host) # Subscription scenarios if fanout: sock_type = zmq.SUB subscribe = ('', fanout)[type(fanout) == str] topic = 'fanout~' + topic.split('.', 1)[0] else: sock_type = zmq.PULL subscribe = None topic = '.'.join((topic.split('.', 1)[0], CONF.rpc_zmq_host)) if topic in self.topics: LOG.info(_("Skipping topic registration. Already registered.")) return # Receive messages from (local) proxy inaddr = "ipc://%s/zmq_topic_%s" % \ (CONF.rpc_zmq_ipc_dir, topic) LOG.debug(_("Consumer is a zmq.%s"), ['PULL', 'SUB'][sock_type == zmq.SUB]) self.reactor.register(proxy, inaddr, sock_type, subscribe=subscribe, in_bind=False) self.topics.append(topic)
def publisher(waiter): LOG.info(_("Creating proxy for topic: %s"), topic) try: # The topic is received over the network, # don't trust this input. if self.badchars.search(topic) is not None: emsg = _("Topic contained dangerous characters.") LOG.warn(emsg) raise RPCException(emsg) out_sock = ZmqSocket("ipc://%s/zmq_topic_%s" % (ipc_dir, topic), sock_type, bind=True) except RPCException: waiter.send_exception(*sys.exc_info()) return self.topic_proxy[topic] = eventlet.queue.LightQueue( CONF.rpc_zmq_topic_backlog) self.sockets.append(out_sock) # It takes some time for a pub socket to open, # before we can have any faith in doing a send() to it. if sock_type == zmq.PUB: eventlet.sleep(.5) waiter.send(True) while(True): data = self.topic_proxy[topic].get() out_sock.send(data, copy=False)
def _parse_check(rule): """ Parse a single base check rule into an appropriate Check object. """ # Handle the special checks if rule == '!': return FalseCheck() elif rule == '@': return TrueCheck() try: kind, match = rule.split(':', 1) except Exception: LOG.exception(_("Failed to understand rule %s") % rule) # If the rule is invalid, we'll fail closed return FalseCheck() # Find what implements the check if kind in _checks: return _checks[kind](kind, match) elif None in _checks: return _checks[None](kind, match) else: LOG.error(_("No handler for matches of kind %s") % kind) return FalseCheck()
def _process_data(self, ctxt, version, method, namespace, args): """Process a message in a new thread. If the proxy object we have has a dispatch method (see rpc.dispatcher.RpcDispatcher), pass it the version, method, and args and let it dispatch as appropriate. If not, use the old behavior of magically calling the specified method on the proxy we have here. """ ctxt.update_store() try: rval = self.proxy.dispatch(ctxt, version, method, namespace, **args) # Check if the result was a generator if inspect.isgenerator(rval): for x in rval: ctxt.reply(x, None, connection_pool=self.connection_pool) else: ctxt.reply(rval, None, connection_pool=self.connection_pool) # This final None tells multicall that it is done. ctxt.reply(ending=True, connection_pool=self.connection_pool) except rpc_common.ClientException as e: LOG.debug(_('Expected exception during message handling (%s)') % e._exc_info[1]) ctxt.reply(None, e._exc_info, connection_pool=self.connection_pool, log_failure=False) except Exception: # sys.exc_info() is deleted by LOG.exception(). exc_info = sys.exc_info() LOG.error(_('Exception during message handling'), exc_info=exc_info) ctxt.reply(None, exc_info, connection_pool=self.connection_pool)
def __call__(self, message_data): """Consumer callback to call a method on a proxy object. Parses the message for validity and fires off a thread to call the proxy object method. Message data should be a dictionary with two keys: method: string representing the method to call args: dictionary of arg: value Example: {'method': 'echo', 'args': {'value': 42}} """ # It is important to clear the context here, because at this point # the previous context is stored in local.store.context if hasattr(local.store, 'context'): del local.store.context rpc_common._safe_log(LOG.debug, _('received %s'), message_data) self.msg_id_cache.check_duplicate_message(message_data) ctxt = unpack_context(self.conf, message_data) method = message_data.get('method') args = message_data.get('args', {}) version = message_data.get('version') namespace = message_data.get('namespace') if not method: LOG.warn(_('no method for message: %s') % message_data) ctxt.reply(_('No method for message: %s') % message_data, connection_pool=self.connection_pool) return self.pool.spawn_n(self._process_data, ctxt, version, method, namespace, args)
def validate_integer(value, property_name, min_val=None, max_val=None): if isinstance(value, basestring): try: value = int(value) except ValueError: pass value = __validate_type(value, property_name, (int, long), "Integer") if min_val is not None and value < min_val: raise ValidationError( _("'%(property_name)s' property value[%(property_value)s] is less " "then min_value[%(min_value)s]."), property_name=property_name, property_value=value, min_value=min_val ) if max_val is not None and value > max_val: raise ValidationError( _("'%(property_name)s' property value[%(property_value)s] is more " "then max_value[%(max_value)s]."), property_name=property_name, property_value=value, max_value=max_val ) return value
def wait(self): """Loop waiting on children to die and respawning as necessary.""" LOG.debug(_('Full set of CONF:')) CONF.log_opt_values(LOG, std_logging.DEBUG) while True: self.handle_signal() self._respawn_children() if self.sigcaught: signame = {signal.SIGTERM: 'SIGTERM', signal.SIGINT: 'SIGINT', signal.SIGHUP: 'SIGHUP'}[self.sigcaught] LOG.info(_('Caught %s, stopping children'), signame) if self.sigcaught != signal.SIGHUP: break for pid in self.children: os.kill(pid, signal.SIGHUP) self.running = True self.sigcaught = None for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def _start_child(self, wrap): if len(wrap.forktimes) > wrap.workers: # Limit ourselves to one process a second (over the period of # number of workers * 1 second). This will allow workers to # start up quickly but ensure we don't fork off children that # die instantly too quickly. if time.time() - wrap.forktimes[0] < wrap.workers: LOG.info(_('Forking too fast, sleeping')) time.sleep(1) wrap.forktimes.pop(0) wrap.forktimes.append(time.time()) pid = os.fork() if pid == 0: # NOTE(johannes): All exceptions are caught to ensure this # doesn't fallback into the loop spawning children. It would # be bad for a child to spawn more children. launcher = self._child_process(wrap.service) while True: self._child_process_handle_signal() status, signo = self._child_wait_for_exit_or_signal(launcher) if signo != signal.SIGHUP: break launcher.restart() os._exit(status) LOG.info(_('Started child %d'), pid) wrap.children.add(pid) self.children[pid] = wrap return pid
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: idle = self.f(*self.args, **self.kw) if not self._running: break if periodic_interval_max is not None: idle = min(idle, periodic_interval_max) LOG.debug( _('Dynamic looping call sleeping for %.02f ' 'seconds'), idle) greenthread.sleep(idle) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in dynamic looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def __call__(self, message_data): """Consumer callback to call a method on a proxy object. Parses the message for validity and fires off a thread to call the proxy object method. Message data should be a dictionary with two keys: method: string representing the method to call args: dictionary of arg: value Example: {'method': 'echo', 'args': {'value': 42}} """ # It is important to clear the context here, because at this point # the previous context is stored in local.store.context if hasattr(local.store, 'context'): del local.store.context rpc_common._safe_log(LOG.debug, 'received %s', message_data) self.msg_id_cache.check_duplicate_message(message_data) ctxt = unpack_context(self.conf, message_data) method = message_data.get('method') args = message_data.get('args', {}) version = message_data.get('version') namespace = message_data.get('namespace') if not method: LOG.warn(_('no method for message: %s') % message_data) ctxt.reply(_('No method for message: %s') % message_data, connection_pool=self.connection_pool) return self.pool.spawn_n(self._process_data, ctxt, version, method, namespace, args)
def __init__(self, addr, zmq_type, bind=True, subscribe=None): self.sock = _get_ctxt().socket(zmq_type) self.addr = addr self.type = zmq_type self.subscriptions = [] # Support failures on sending/receiving on wrong socket type. self.can_recv = zmq_type in (zmq.PULL, zmq.SUB) self.can_send = zmq_type in (zmq.PUSH, zmq.PUB) self.can_sub = zmq_type in (zmq.SUB, ) # Support list, str, & None for subscribe arg (cast to list) do_sub = { list: subscribe, str: [subscribe], type(None): [] }[type(subscribe)] for f in do_sub: self.subscribe(f) str_data = {'addr': addr, 'type': self.socket_s(), 'subscribe': subscribe, 'bind': bind} LOG.debug(_("Connecting to %(addr)s with %(type)s"), str_data) LOG.debug(_("-> Subscribed to %(subscribe)s"), str_data) LOG.debug(_("-> bind: %(bind)s"), str_data) try: if bind: self.sock.bind(addr) else: self.sock.connect(addr) except Exception: raise RPCException(_("Could not open socket."))
def _multi_send(method, context, topic, msg, timeout=None, envelope=False, _msg_id=None): """Wraps the sending of messages. Dispatches to the matchmaker and sends message to all relevant hosts. """ conf = CONF LOG.debug("%(msg)s" % {'msg': ' '.join(map(pformat, (topic, msg)))}) queues = _get_matchmaker().queues(topic) LOG.debug("Sending message(s) to: %s", queues) # Don't stack if we have no matchmaker results if not queues: LOG.warn(_("No matchmaker results. Not casting.")) # While not strictly a timeout, callers know how to handle # this exception and a timeout isn't too big a lie. raise rpc_common.Timeout(_("No match from matchmaker.")) # This supports brokerless fanout (addresses > 1) for queue in queues: (_topic, ip_addr) = queue _addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port) if method.__name__ == '_cast': eventlet.spawn_n(method, _addr, context, _topic, msg, timeout, envelope, _msg_id) return return method(_addr, context, _topic, msg, timeout, envelope)
def parse_key_schema(cls, key_def_list_json): hash_key_attr_name = None range_key_attr_name = None for key_def in key_def_list_json: key_attr_name_json = key_def.pop(Props.ATTRIBUTE_NAME, None) validation.validate_attr_name(key_attr_name_json) key_type_json = key_def.pop(Props.KEY_TYPE, None) if key_type_json == Values.KEY_TYPE_HASH: if hash_key_attr_name is not None: raise ValidationError(_("Only one 'HASH' key is allowed")) hash_key_attr_name = key_attr_name_json elif key_type_json == Values.KEY_TYPE_RANGE: if range_key_attr_name is not None: raise ValidationError(_("Only one 'RANGE' key is allowed")) range_key_attr_name = key_attr_name_json else: raise ValidationError( _("Only 'RANGE' or 'HASH' key types are allowed, but " "'%(key_type)s' is found"), key_type=key_type_json) validation.validate_unexpected_props(key_def, "key_definition") if hash_key_attr_name is None: raise ValidationError(_("HASH key is missing")) if range_key_attr_name: return (hash_key_attr_name, range_key_attr_name) return (hash_key_attr_name,)
def _wait_child(self): try: # Don't block if no child processes have exited pid, status = os.waitpid(0, os.WNOHANG) if not pid: return None except OSError as exc: if exc.errno not in (errno.EINTR, errno.ECHILD): raise return None if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) LOG.info(_('Child %(pid)d killed by signal %(sig)d'), dict(pid=pid, sig=sig)) else: code = os.WEXITSTATUS(status) LOG.info(_('Child %(pid)s exited with status %(code)d'), dict(pid=pid, code=code)) if pid not in self.children: LOG.warning(_('pid %d not in child list'), pid) return None wrap = self.children.pop(pid) wrap.children.remove(pid) return wrap
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def parse_key_schema(cls, key_def_list_json): hash_key_attr_name = None range_key_attr_name = None for key_def in key_def_list_json: key_attr_name_json = key_def.pop(Props.ATTRIBUTE_NAME, None) validation.validate_attr_name(key_attr_name_json) key_type_json = key_def.pop(Props.KEY_TYPE, None) if key_type_json == Values.KEY_TYPE_HASH: if hash_key_attr_name is not None: raise ValidationError(_("Only one 'HASH' key is allowed")) hash_key_attr_name = key_attr_name_json elif key_type_json == Values.KEY_TYPE_RANGE: if range_key_attr_name is not None: raise ValidationError(_("Only one 'RANGE' key is allowed")) range_key_attr_name = key_attr_name_json else: raise ValidationError(_( "Only 'RANGE' or 'HASH' key types are allowed, but " "'%(key_type)s' is found"), key_type=key_type_json) validation.validate_unexpected_props(key_def, "key_definition") if hash_key_attr_name is None: raise ValidationError(_("HASH key is missing")) if range_key_attr_name: return (hash_key_attr_name, range_key_attr_name) return (hash_key_attr_name, )
def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" for task_name, task in self._periodic_tasks: full_task_name = '.'.join([self.__class__.__name__, task_name]) ticks_to_skip = self._ticks_to_skip[task_name] if ticks_to_skip > 0: LOG.debug(_("Skipping %(full_task_name)s, %(ticks_to_skip)s" " ticks left until next run"), dict(full_task_name=full_task_name, ticks_to_skip=ticks_to_skip)) self._ticks_to_skip[task_name] -= 1 continue self._ticks_to_skip[task_name] = task._ticks_between_runs LOG.debug(_("Running periodic task %(full_task_name)s"), dict(full_task_name=full_task_name)) try: task(self, context) except Exception as e: if raise_on_error: raise LOG.exception(_("Error during %(full_task_name)s:" " %(e)s"), dict(e=e, full_task_name=full_task_name))
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn( _('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def _error_callback(exc): if isinstance(exc, qpid_exceptions.Empty): LOG.debug(_('Timed out waiting for RPC response: %s') % str(exc)) raise rpc_common.Timeout() else: LOG.exception(_('Failed to consume message from queue: %s') % str(exc))
def _error_callback(exc): if isinstance(exc, socket.timeout): LOG.debug(_('Timed out waiting for RPC response: %s') % str(exc)) raise rpc_common.Timeout() else: LOG.exception(_('Failed to consume message from queue: %s') % str(exc)) info['do_consume'] = True
def _process_data(self, message_data): msg_id = message_data.pop('_msg_id', None) waiter = self._call_waiters.get(msg_id) if not waiter: LOG.warn(_('No calling threads waiting for msg_id : %(msg_id)s' ', message : %(data)s'), {'msg_id': msg_id, 'data': message_data}) LOG.warn(_('_call_waiters: %s') % str(self._call_waiters)) else: waiter.put(message_data)
def parse_attribute_condition(cls, condition_type, condition_args, condition_class=IndexedCondition): actual_args_count = ( len(condition_args) if condition_args is not None else 0 ) if condition_type == Values.BETWEEN: if actual_args_count != 2: raise ValidationError( _("%(type)s condition type requires exactly 2 arguments, " "but %(actual_args_count)s given"), type=condition_type, actual_args_count=actual_args_count ) if condition_args[0].attr_type != condition_args[1].attr_type: raise ValidationError( _("%(type)s condition type requires arguments of the " "same type, but different types given"), type=condition_type, ) return [ condition_class.ge(condition_args[0]), condition_class.le(condition_args[1]) ] if condition_type == Values.BEGINS_WITH: first_condition = condition_class( condition_class.CONDITION_TYPE_GREATER_OR_EQUAL, condition_args ) condition_arg = first_condition.arg if condition_arg.is_number: raise ValidationError( _("%(condition_type)s condition type is not allowed for" "argument of the %(argument_type)s type"), condition_type=condition_type, argument_type=condition_arg.attr_type.type ) first_value = condition_arg.decoded_value chr_fun = unichr if isinstance(first_value, unicode) else chr second_value = first_value[:-1] + chr_fun(ord(first_value[-1]) + 1) second_condition = condition_class.le( AttributeValue( condition_arg.attr_type, decoded_value=second_value ) ) return [first_condition, second_condition] return [condition_class(condition_type, condition_args)]
def notify(context, publisher_id, event_type, priority, payload): """Sends a notification using the specified driver :param publisher_id: the source worker_type.host of the message :param event_type: the literal type of event (ex. Instance Creation) :param priority: patterned after the enumeration of Python logging levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) :param payload: A python dictionary of attributes Outgoing message format includes the above parameters, and appends the following: message_id a UUID representing the id for this notification timestamp the GMT timestamp the notification was sent at The composite message will be constructed as a dictionary of the above attributes, which will then be sent via the transport mechanism defined by the driver. Message example:: {'message_id': str(uuid.uuid4()), 'publisher_id': 'compute.host1', 'timestamp': timeutils.utcnow(), 'priority': 'WARN', 'event_type': 'compute.create_instance', 'payload': {'instance_id': 12, ... }} """ if priority not in log_levels: raise BadPriorityException( _('%s not in valid priorities') % priority) # Ensure everything is JSON serializable. payload = jsonutils.to_primitive(payload, convert_instances=True) msg = dict(message_id=str(uuid.uuid4()), publisher_id=publisher_id, event_type=event_type, priority=priority, payload=payload, timestamp=str(timeutils.utcnow())) for driver in _get_drivers(): try: driver.notify(context, msg) except Exception as e: LOG.exception(_("Problem '%(e)s' attempting to " "send to notification system. " "Payload=%(payload)s") % dict(e=e, payload=payload))
def notify(context, event_type, payload, priority=None): assert __PUBLISHER_ID is not None priority = priority or __DEFAULT_PRIORITY if event_type not in __ALLOWED_EVENT_TYPES: raise BadEventTypeException( _('%s is not a valid event type') % event_type) if priority not in __ALLOWED_PRIORITIES: raise notifier_api.BadPriorityException( _('%s not in valid priorities') % priority) notifier_api.notify(context, __PUBLISHER_ID, event_type, priority, payload)
def reconnect(self): """Handles reconnecting and re-establishing queues. Will retry up to self.max_retries number of times. self.max_retries = 0 means to retry forever. Sleep between tries, starting at self.interval_start seconds, backing off self.interval_stepping number of seconds each attempt. """ attempt = 0 while True: params = self.params_list[attempt % len(self.params_list)] attempt += 1 try: self._connect(params) return except (IOError, self.connection_errors) as e: pass except Exception as e: # NOTE(comstud): Unfortunately it's possible for amqplib # to return an error not covered by its transport # connection_errors in the case of a timeout waiting for # a protocol response. (See paste link in LP888621) # So, we check all exceptions for 'timeout' in them # and try to reconnect in this case. if 'timeout' not in str(e): raise log_info = {} log_info['err_str'] = str(e) log_info['max_retries'] = self.max_retries log_info.update(params) if self.max_retries and attempt == self.max_retries: msg = _('Unable to connect to AMQP server on ' '%(hostname)s:%(port)d after %(max_retries)d ' 'tries: %(err_str)s') % log_info LOG.error(msg) raise rpc_common.RPCException(msg) if attempt == 1: sleep_time = self.interval_start or 1 elif attempt > 1: sleep_time += self.interval_stepping if self.interval_max: sleep_time = min(sleep_time, self.interval_max) log_info['sleep_time'] = sleep_time LOG.error(_('AMQP server on %(hostname)s:%(port)d is ' 'unreachable: %(err_str)s. Trying again in ' '%(sleep_time)d seconds.') % log_info) time.sleep(sleep_time)
def _process_data(self, message_data): msg_id = message_data.pop('_msg_id', None) waiter = self._call_waiters.get(msg_id) if not waiter: LOG.warn( _('No calling threads waiting for msg_id : %(msg_id)s' ', message : %(data)s'), { 'msg_id': msg_id, 'data': message_data }) LOG.warn(_('_call_waiters: %s') % str(self._call_waiters)) else: waiter.put(message_data)
def parse_local_secondary_index(cls, local_secondary_index_json): key_attrs_json = local_secondary_index_json.pop(Props.KEY_SCHEMA, None) validation.validate_list(key_attrs_json, Props.KEY_SCHEMA) key_attrs_for_projection = cls.parse_key_schema(key_attrs_json) hash_key = key_attrs_for_projection[0] try: range_key = key_attrs_for_projection[1] except IndexError: raise ValidationError(_("Range key in index wasn't specified")) index_name = local_secondary_index_json.pop(Props.INDEX_NAME, None) validation.validate_index_name(index_name) projection_json = local_secondary_index_json.pop(Props.PROJECTION, None) validation.validate_object(projection_json, Props.PROJECTION) validation.validate_unexpected_props( local_secondary_index_json, "local_secondary_index" ) projection_type = projection_json.pop( Props.PROJECTION_TYPE, Values.PROJECTION_TYPE_INCLUDE ) if projection_type == Values.PROJECTION_TYPE_ALL: projected_attrs = None elif projection_type == Values.PROJECTION_TYPE_KEYS_ONLY: projected_attrs = tuple() elif projection_type == Values.PROJECTION_TYPE_INCLUDE: projected_attrs = projection_json.pop( Props.NON_KEY_ATTRIBUTES, None ) else: raise ValidationError( _("Only '%(pt_all)', '%(pt_ko)' of '%(pt_incl)' projection " "types are allowed, but '%(projection_type)s' is found"), pt_all=Values.PROJECTION_TYPE_ALL, pt_ko=Values.PROJECTION_TYPE_KEYS_ONLY, pt_incl=Values.PROJECTION_TYPE_INCLUDE, projection_type=projection_type ) validation.validate_unexpected_props(projection_json, Props.PROJECTION) return index_name, IndexDefinition( hash_key, range_key, projected_attrs )
def __init__(self, select_type, attributes=None): if select_type not in self._allowed_types: raise ValidationError( _("Select type '%(select_type)s' isn't allowed"), select_type=select_type) if attributes is not None: if select_type != self.SELECT_TYPE_SPECIFIC: raise ValidationError(_( "Attribute list is only expected with select_type " "'%(select_type)s'"), select_type=self.SELECT_TYPE_SPECIFIC) super(SelectType, self).__init__(type=select_type, attributes=attributes)
def __init__(self, info=None, topic=None, method=None): """Initiates Timeout object. :param info: Extra info to convey to the user :param topic: The topic that the rpc call was sent to :param rpc_method_name: The name of the rpc method being called """ self.info = info self.topic = topic self.method = method super(Timeout, self).__init__(None, info=info or _('<unknown>'), topic=topic or _('<unknown>'), method=method or _('<unknown>'))
class Timeout(RPCException): """Signifies that a timeout has occurred. This exception is raised if the rpc_response_timeout is reached while waiting for a response from the remote side. """ msg_fmt = _('Timeout while waiting on RPC response - ' 'topic: "%(topic)s", RPC method: "%(method)s" ' 'info: "%(info)s"') def __init__(self, info=None, topic=None, method=None): """Initiates Timeout object. :param info: Extra info to convey to the user :param topic: The topic that the rpc call was sent to :param rpc_method_name: The name of the rpc method being called """ self.info = info self.topic = topic self.method = method super(Timeout, self).__init__(None, info=info or _('<unknown>'), topic=topic or _('<unknown>'), method=method or _('<unknown>'))
def __decode_value(cls, attr_type, encoded_value): decoded_value = None if decoded_value is not None: return decoded_value collection_type = attr_type.collection_type if collection_type is None: decoded_value = cls.__decode_single_value(attr_type.type, encoded_value) elif collection_type == AttributeType.COLLECTION_TYPE_MAP: if isinstance(encoded_value, dict): res_dict = dict() key_type = attr_type.key_type value_type = attr_type.value_type for key, value in encoded_value.iteritems(): res_dict[cls.__decode_single_value( key_type, key)] = (cls.__decode_single_value(value_type, value)) decoded_value = res_dict elif collection_type == AttributeType.COLLECTION_TYPE_SET: element_type = attr_type.element_type res = sortedset() for val in encoded_value: res.add(cls.__decode_single_value(element_type, val)) decoded_value = res if decoded_value is None: raise ValidationError(_( "Can't recognize attribute value '%(value)s'" "of type %(type)s"), type=attr_type, value=json.dumps(encoded_value)) return decoded_value
def parse_batch_write_request_items(cls, request_items_json): request_map = {} for table_name, request_list_json in request_items_json.iteritems(): validation.validate_table_name(table_name) validation.validate_list_of_objects(request_list_json, table_name) request_list_for_table = [] for request_json in request_list_json: for request_type, request_body in request_json.iteritems(): validation.validate_string(request_type, "request_type") if request_type == Props.REQUEST_PUT: validation.validate_object(request_body, request_type) item = request_body.pop(Props.ITEM, None) validation.validate_object(item, Props.ITEM) validation.validate_unexpected_props( request_body, request_type) request_list_for_table.append( WriteItemRequest.put( cls.parse_item_attributes(item))) elif request_type == Props.REQUEST_DELETE: validation.validate_object(request_body, request_type) key = request_body.pop(Props.KEY, None) validation.validate_object(key, Props.KEY) validation.validate_unexpected_props( request_body, request_type) request_list_for_table.append( WriteItemRequest.delete( cls.parse_item_attributes(key))) else: raise ValidationError(_( "Unsupported request type found: " "%(request_type)s"), request_type=request_type) request_map[table_name] = request_list_for_table return request_map
def bool_from_string(subject, strict=False, default=False): """Interpret a string as a boolean. A case-insensitive match is performed such that strings matching 't', 'true', 'on', 'y', 'yes', or '1' are considered True and, when `strict=False`, anything else returns the value specified by 'default'. Useful for JSON-decoded stuff and config file parsing. If `strict=True`, unrecognized values, including None, will raise a ValueError which is useful when parsing values passed in from an API call. Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. """ if not isinstance(subject, six.string_types): subject = six.text_type(subject) lowered = subject.strip().lower() if lowered in TRUE_STRINGS: return True elif lowered in FALSE_STRINGS: return False elif strict: acceptable = ', '.join("'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) msg = _("Unrecognized value '%(val)s', acceptable values are:" " %(acceptable)s") % { 'val': subject, 'acceptable': acceptable } raise ValueError(msg) else: return default
def deprecated(self, msg, *args, **kwargs): stdmsg = _("Deprecated: %s") % msg if CONF.fatal_deprecations: self.critical(stdmsg, *args, **kwargs) raise DeprecatedConfig(msg=stdmsg) else: self.warn(stdmsg, *args, **kwargs)
def _raise_condition_schema_mismatch(condition_map, table_info): raise ValidationError( _("Specified query conditions %(indexed_condition_map)s " "don't match table schema: %(table_schema)s"), indexed_condition_map=condition_map, table_schema=table_info.schema )
def fanout_cast(conf, context, topic, msg, connection_pool): """Sends a message on a fanout exchange without waiting for a response.""" LOG.debug(_('Making asynchronous fanout cast...')) _add_unique_id(msg) pack_context(msg, context) with ConnectionContext(conf, connection_pool) as conn: conn.fanout_send(topic, rpc_common.serialize_msg(msg))
def start(self): super(Service, self).start() self.conn = rpc.create_connection(new=True) LOG.debug(_("Creating Consumer connection for Service %s") % self.topic) dispatcher = rpc_dispatcher.RpcDispatcher([self.manager], self.serializer) # Share this same connection for these Consumers self.conn.create_consumer(self.topic, dispatcher, fanout=False) node_topic = '%s.%s' % (self.topic, self.host) self.conn.create_consumer(node_topic, dispatcher, fanout=False) self.conn.create_consumer(self.topic, dispatcher, fanout=True) # Hook to allow the manager to do other initializations after # the rpc connection is created. if callable(getattr(self.manager, 'initialize_service_hook', None)): self.manager.initialize_service_hook(self) # Consume from all consumers in a thread self.conn.consume_in_thread()
def validate_index_name(value): validate_string(value, "index name") if not INDEX_NAME_PATTERN.match(value): raise ValidationError(_("Wrong index name '%(prop_value)s' found"), prop_value=value) return value
def __init__(self, info=None, topic=None, method=None): """Initiates Timeout object. :param info: Extra info to convey to the user :param topic: The topic that the rpc call was sent to :param rpc_method_name: The name of the rpc method being called """ self.info = info self.topic = topic self.method = method super(Timeout, self).__init__( None, info=info or _('<unknown>'), topic=topic or _('<unknown>'), method=method or _('<unknown>'))
def publisher(waiter): LOG.info(_LI("Creating proxy for topic: %s"), topic) try: # The topic is received over the network, # don't trust this input. if self.badchars.search(topic) is not None: emsg = _("Topic contained dangerous characters.") LOG.warn(emsg) raise RPCException(emsg) out_sock = ZmqSocket("ipc://%s/zmq_topic_%s" % (ipc_dir, topic), sock_type, bind=True) except RPCException: waiter.send_exception(*sys.exc_info()) return self.topic_proxy[topic] = eventlet.queue.LightQueue( CONF.rpc_zmq_topic_backlog) self.sockets.append(out_sock) # It takes some time for a pub socket to open, # before we can have any faith in doing a send() to it. if sock_type == zmq.PUB: eventlet.sleep(.5) waiter.send(True) while (True): data = self.topic_proxy[topic].get() out_sock.send(data, copy=False)
def validate_table_name(value): validate_string(value, "table name") if not TABLE_NAME_PATTERN.match(value): raise ValidationError(_("Wrong table name '%(prop_value)s' found"), prop_value=value) return value
def validate_unexpected_props(value, property_name): if len(value) > 0: raise ValidationError(_( "Unexpected properties were found for '%(property_name)s': " "%(unexpected_props)s"), property_name=property_name, unexpected_props=json.dumps(value)) return value
def validate_attr_name(value): validate_string(value, "attribute name") if not ATTRIBUTE_NAME_PATTERN.match(value): raise ValidationError( _("Wrong attribute name '%(prop_value)s' found"), prop_value=value )
def parse_typed_attr_value(cls, typed_attr_value_json): if len(typed_attr_value_json) != 1: raise ValidationError( _("Can't recognize attribute typed value format: '%(attr)s'"), attr=json.dumps(typed_attr_value_json)) (attr_type_json, attr_value_json) = (typed_attr_value_json.popitem()) return AttributeValue(attr_type_json, attr_value_json)
def __call__(self, req): if not self._conf_get('multi_cloud'): return self._authorize(req, self._conf_get_auth_uri()) else: # attempt to authorize for each configured allowed_auth_uris # until one is successful. # This is safe for the following reasons: # 1. AWSAccessKeyId is a randomly generated sequence # 2. No secret is transferred to validate a request last_failure = None for auth_uri in self._conf_get('allowed_auth_uris'): try: logger.debug(_("Attempt authorize on %s") % auth_uri) return self._authorize(req, auth_uri) except exception.AWSErrorResponseException as e: logger.debug(_("Authorize failed: %s") % e.__class__) last_failure = e raise last_failure or exception.AccessDeniedError()
def _validate_table_is_active(table_info): if table_info.status != TableMeta.TABLE_STATUS_ACTIVE: raise ValidationError( _("Can't execute request: " "Table '%(table_name)s' status '%(table_status)s' " "isn't %(expected_status)s"), table_name=table_info.name, table_status=table_info.status, expected_status=TableMeta.TABLE_STATUS_ACTIVE )
def __call__(self, request): """WSGI method that controls (de)serialization and method dispatch.""" try: action, action_args, accept = self.deserialize_request(request) except exception.InvalidContentType: msg = _("Unsupported Content-Type") return webob.exc.HTTPUnsupportedMediaType(explanation=msg) except exception.MalformedRequestBody: msg = _("Malformed request body") return webob.exc.HTTPBadRequest(explanation=msg) action_result = self.execute_action(action, request, **action_args) try: return self.serialize_response(action, action_result, accept) # return unserializable result (typically a webob exc) except Exception: return action_result
def _from_xml(self, datastring): plurals = set(self.metadata.get('plurals', {})) try: node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0] return {node.nodeName: self._from_xml_node(node, plurals)} except expat.ExpatError: msg = _("cannot understand XML") raise exception.MalformedRequestBody(reason=msg)
def add_call_waiter(self, waiter, msg_id): self._num_call_waiters += 1 if self._num_call_waiters > self._num_call_waiters_wrn_threshold: LOG.warn( _('Number of call waiters is greater than warning ' 'threshold: %d. There could be a MulticallProxyWaiter ' 'leak.') % self._num_call_waiters_wrn_threshold) self._num_call_waiters_wrn_threshold *= 2 self._call_waiters[msg_id] = waiter