def check_string_length(value, name, min_length=0, max_length=None): """Check the length of specified string. :param value: the value of the string :param name: the name of the string :param min_length: the min_length of the string :param max_length: the max_length of the string """ if not isinstance(value, six.string_types): msg = _("%s is not a string or unicode") % name raise exception.InvalidInput(message=msg) if len(value) < min_length: msg = _("%(name)s has a minimum character requirement of " "%(min_length)s.") % { 'name': name, 'min_length': min_length } raise exception.InvalidInput(message=msg) if max_length and len(value) > max_length: msg = _("%(name)s has more than %(max_length)s " "characters.") % { 'name': name, 'max_length': max_length } raise exception.InvalidInput(message=msg)
def validate_integer(value, name, min_value=None, max_value=None): """Make sure that value is a valid integer, potentially within range. :param value: the value of the integer :param name: the name of the integer :param min_length: the min_length of the integer :param max_length: the max_length of the integer :returns: integer """ try: value = int(value) except (TypeError, ValueError, UnicodeEncodeError): raise webob.exc.HTTPBadRequest( explanation=(_('%s must be an integer.') % name)) if min_value is not None and value < min_value: raise webob.exc.HTTPBadRequest( explanation=(_('%(value_name)s must be >= %(min_value)d') % { 'value_name': name, 'min_value': min_value })) if max_value is not None and value > max_value: raise webob.exc.HTTPBadRequest( explanation=(_('%(value_name)s must be <= %(max_value)d') % { 'value_name': name, 'max_value': max_value })) return value
def __init__(self, message=None, **kwargs): if kwargs.get('host', None): self.message = _("Service %(service_id)s could not be " "found on host %(host)s.") else: self.message = _("Service %(service_id)s could not be found.") super(ServiceNotFound, self).__init__(None, **kwargs)
def save(self): updates = self.waterfall_obj_get_changes() if updates: if 'consistencygroup' in updates: raise exception.ObjectActionError( action='save', reason=_('consistencygroup changed')) if 'glance_metadata' in updates: raise exception.ObjectActionError( action='save', reason=_('glance_metadata changed')) if 'snapshots' in updates: raise exception.ObjectActionError( action='save', reason=_('snapshots changed')) if 'metadata' in updates: # Metadata items that are not specified in the # self.metadata will be deleted metadata = updates.pop('metadata', None) self.metadata = db.workflow_metadata_update( self._context, self.id, metadata, True) if self._context.is_admin and 'admin_metadata' in updates: metadata = updates.pop('admin_metadata', None) self.admin_metadata = db.workflow_admin_metadata_update( self._context, self.id, metadata, True) db.workflow_update(self._context, self.id, updates) self.obj_reset_changes()
def save(self): updates = self.waterfall_obj_get_changes() if updates: if 'consistencygroup' in updates: raise exception.ObjectActionError( action='save', reason=_('consistencygroup changed')) if 'snapshots' in updates: raise exception.ObjectActionError( action='save', reason=_('snapshots changed')) db.cgsnapshot_update(self._context, self.id, updates) self.obj_reset_changes()
def create(self): try: ssh = paramiko.SSHClient() if ',' in self.hosts_key_file: files = self.hosts_key_file.split(',') for f in files: ssh.load_host_keys(f) else: ssh.load_host_keys(self.hosts_key_file) # If strict_ssh_host_key_policy is set we want to reject, by # default if there is not entry in the known_hosts file. # Otherwise we use AutoAddPolicy which accepts on the first # Connect but fails if the keys change. load_host_keys can # handle hashed known_host entries. if self.strict_ssh_host_key_policy: ssh.set_missing_host_key_policy(paramiko.RejectPolicy()) else: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if self.password: ssh.connect(self.ip, port=self.port, username=self.login, password=self.password, timeout=self.conn_timeout) elif self.privatekey: pkfile = os.path.expanduser(self.privatekey) privatekey = paramiko.RSAKey.from_private_key_file(pkfile) ssh.connect(self.ip, port=self.port, username=self.login, pkey=privatekey, timeout=self.conn_timeout) else: msg = _("Specify a password or private_key") raise exception.WaterfallException(msg) # Paramiko by default sets the socket timeout to 0.1 seconds, # ignoring what we set through the sshclient. This doesn't help for # keeping long lived connections. Hence we have to bypass it, by # overriding it after the transport is initialized. We are setting # the sockettimeout to None and setting a keepalive packet so that, # the server will keep the connection open. All that does is send # a keepalive packet every ssh_conn_timeout seconds. if self.conn_timeout: transport = ssh.get_transport() transport.sock.settimeout(None) transport.set_keepalive(self.conn_timeout) return ssh except Exception as e: msg = _("Error connecting via ssh: %s") % six.text_type(e) LOG.error(msg) raise paramiko.SSHException(msg)
def create(self): if self.obj_attr_is_set('id'): raise exception.ObjectActionError(action='create', reason=_('already_created')) updates = self.waterfall_obj_get_changes() if 'consistencygroup' in updates: raise exception.ObjectActionError( action='create', reason=_('consistencygroup assigned')) db_cgsnapshots = db.cgsnapshot_create(self._context, updates) self._from_db_object(self._context, self, db_cgsnapshots)
def decode_record(backup_url): """Deserialize backup metadata from string into a dictionary. :raises: InvalidInput """ try: return jsonutils.loads(base64.decode_as_text(backup_url)) except TypeError: msg = _("Can't decode backup record.") except ValueError: msg = _("Can't parse backup record.") raise exception.InvalidInput(reason=msg)
def create(self): if self.obj_attr_is_set('id'): raise exception.ObjectActionError(action='create', reason=_('already created')) updates = self.waterfall_obj_get_changes() if 'workflow' in updates: raise exception.ObjectActionError(action='create', reason=_('workflow assigned')) if 'cgsnapshot' in updates: raise exception.ObjectActionError(action='create', reason=_('cgsnapshot assigned')) db_snapshot = db.snapshot_create(self._context, updates) self._from_db_object(self._context, self, db_snapshot)
class WorkflowLimitExceeded(QuotaError): message = _("Maximum number of workflows allowed (%(allowed)d) exceeded for " "quota '%(name)s'.") def __init__(self, message=None, **kwargs): kwargs.setdefault('name', 'workflows') super(WorkflowLimitExceeded, self).__init__(message, **kwargs)
class APIException(WaterfallException): message = _("Error while requesting %(service)s API.") def __init__(self, message=None, **kwargs): if 'service' not in kwargs: kwargs['service'] = 'unknown' super(APIException, self).__init__(message, **kwargs)
def __init__(self, name, loader=None): """Initialize, but do not start the WSGI server. :param name: The name of the WSGI server given to the loader. :param loader: Loads the WSGI application using the given name. :returns: None """ self.name = name self.manager = self._get_manager() self.loader = loader or wsgi.Loader(CONF) self.app = self.loader.load_app(name) self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0") self.port = getattr(CONF, '%s_listen_port' % name, 0) self.workers = (getattr(CONF, '%s_workers' % name, None) or processutils.get_worker_count()) if self.workers and self.workers < 1: worker_name = '%s_workers' % name msg = (_("%(worker_name)s value of %(workers)d is invalid, " "must be greater than 0.") % { 'worker_name': worker_name, 'workers': self.workers }) raise exception.InvalidInput(msg) setup_profiler(name, self.host) self.server = wsgi.Server(CONF, name, self.app, host=self.host, port=self.port)
def _db_error(caught_exception): print('%s' % caught_exception) print( _("The above error may show that the database has not " "been created.\nPlease create a database using " "'waterfall-manage db sync' before running this command.")) sys.exit(1)
def _error(self, inner, req): if not isinstance(inner, exception.QuotaError): LOG.exception(_LE("Caught error: %(type)s %(error)s"), { 'type': type(inner), 'error': inner }) safe = getattr(inner, 'safe', False) headers = getattr(inner, 'headers', None) status = getattr(inner, 'code', 500) if status is None: status = 500 msg_dict = dict(url=req.url, status=status) LOG.info(_LI("%(url)s returned with HTTP %(status)d"), msg_dict) outer = self.status_to_type(status) if headers: outer.headers = headers # NOTE(johannes): We leave the explanation empty here on # purpose. It could possibly have sensitive information # that should not be returned back to the user. See # bugs 868360 and 874472 # NOTE(eglynn): However, it would be over-conservative and # inconsistent with the EC2 API to hide every exception, # including those that are safe to expose, see bug 1021373 if safe: msg = (inner.msg if isinstance(inner, exception.WaterfallException) else six.text_type(inner)) params = { 'exception': inner.__class__.__name__, 'explanation': msg } outer.explanation = _('%(exception)s: %(explanation)s') % params return wsgi.Fault(outer)
def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: # Option 1: simple string res = 'message\n' # Option 2: a nicely formatted HTTP exception page res = exc.HTTPForbidden(explanation='Nice try') # Option 3: a webob Response object (in case you need to play with # headers, or you want to be treated like an iterable) res = Response(); res.app_iter = open('somefile') # Option 4: any wsgi app to be run next res = self.application # Option 5: you can get a Response object for a wsgi app, too, to # play with headers etc res = req.get_response(self.application) # You can then just return your response... return res # ... or set req.response and return None. req.response = res See the end of http://pythonpaste.org/webob/modules/dec.html for more info. """ raise NotImplementedError(_('You must implement __call__'))
def create(self): if self.obj_attr_is_set('id'): raise exception.ObjectActionError(action='create', reason=_('already created')) updates = self.waterfall_obj_get_changes() db_service = db.service_create(self._context, updates) self._from_db_object(self._context, self, db_service)
def check_exclusive_options(**kwargs): """Checks that only one of the provided options is actually not-none. Iterates over all the kwargs passed in and checks that only one of said arguments is not-none, if more than one is not-none then an exception will be raised with the names of those arguments who were not-none. """ if not kwargs: return pretty_keys = kwargs.pop("pretty_keys", True) exclusive_options = {} for (k, v) in kwargs.items(): if v is not None: exclusive_options[k] = True if len(exclusive_options) > 1: # Change the format of the names from pythonic to # something that is more readable. # # Ex: 'the_key' -> 'the key' if pretty_keys: names = [k.replace('_', ' ') for k in kwargs.keys()] else: names = kwargs.keys() names = ", ".join(sorted(names)) msg = (_("May specify only one of %s") % (names)) raise exception.InvalidInput(reason=msg)
def attach(self, *slaves): """Attach one or more slave templates. Attaches one or more slave templates to the master template. Slave templates must have a root element with the same tag as the master template. The slave template's apply() method will be called to determine if the slave should be applied to this master; if it returns False, that slave will be skipped. (This allows filtering of slaves based on the version of the master template.) """ slave_list = [] for slave in slaves: slave = slave.wrap() # Make sure we have a tree match if slave.root.tag != self.root.tag: msg = (_("Template tree mismatch; adding slave %(slavetag)s " "to master %(mastertag)s") % { 'slavetag': slave.root.tag, 'mastertag': self.root.tag }) raise ValueError(msg) # Make sure slave applies to this template if not slave.apply(self): continue slave_list.append(slave) # Add the slaves self.slaves.extend(slave_list)
def get_blkdev_major_minor(path, lookup_for_file=True): """Get 'major:minor' number of block device. Get the device's 'major:minor' number of a block device to control I/O ratelimit of the specified path. If lookup_for_file is True and the path is a regular file, lookup a disk device which the file lies on and returns the result for the device. """ st = os.stat(path) if stat.S_ISBLK(st.st_mode): path, st = _get_disk_of_partition(path, st) return '%d:%d' % (os.major(st.st_rdev), os.minor(st.st_rdev)) elif stat.S_ISCHR(st.st_mode): # No I/O ratelimit control is provided for character devices return None elif lookup_for_file: # lookup the mounted disk which the file lies on out, _err = execute('df', path) devpath = out.split("\n")[1].split()[0] if devpath[0] is not '/': # the file is on a network file system return None return get_blkdev_major_minor(devpath, False) else: msg = _("Unable to get a block device for file \'%s\'") % path raise exception.Error(msg)
def _get_limit_param(params, max_limit=None): """Extract integer limit from request's dictionary or fail. Defaults to max_limit if not present and returns max_limit if present 'limit' is greater than max_limit. """ max_limit = max_limit or CONF.osapi_max_limit try: limit = int(params.pop('limit', max_limit)) except ValueError: msg = _('limit param must be an integer') raise webob.exc.HTTPBadRequest(explanation=msg) if limit < 0: msg = _('limit param must be positive') raise webob.exc.HTTPBadRequest(explanation=msg) limit = min(limit, max_limit) return limit
def purge(self, age_in_days): """Purge deleted rows older than a given age from waterfall tables.""" age_in_days = int(age_in_days) if age_in_days <= 0: print(_("Must supply a positive, non-zero value for age")) sys.exit(1) ctxt = context.get_admin_context() db.purge_deleted_rows(ctxt, age_in_days)
def construct(self): """Construct a template. Called to construct a template instance, which it must return. Only called once. """ raise NotImplementedError(_("subclasses must implement construct()!"))
def create(self): if self.obj_attr_is_set('id'): raise exception.ObjectActionError(action='create', reason=_('already created')) db_workflow_type = workflow_types.create(self._context, self.name, self.extra_specs, self.is_public, self.projects, self.description) self._from_db_object(self._context, self, db_workflow_type)
class WorkflowSizeExceedsAvailableQuota(QuotaError): message = _("Requested workflow or snapshot exceeds allowed %(name)s " "quota. Requested %(requested)sG, quota is %(quota)sG and " "%(consumed)sG has been consumed.") def __init__(self, message=None, **kwargs): kwargs.setdefault('name', 'gigabytes') super(WorkflowSizeExceedsAvailableQuota, self).__init__( message, **kwargs)
class WaterfallException(Exception): """Base Waterfall Exception To correctly use this class, inherit from it and define a 'message' property. That message will get printf'd with the keyword arguments provided to the constructor. """ message = _("An unknown exception occurred.") code = 500 headers = {} safe = False def __init__(self, message=None, **kwargs): self.kwargs = kwargs self.kwargs['message'] = message if 'code' not in self.kwargs: try: self.kwargs['code'] = self.code except AttributeError: pass for k, v in self.kwargs.items(): if isinstance(v, Exception): self.kwargs[k] = six.text_type(v) if self._should_format(): try: message = self.message % kwargs except Exception: exc_info = sys.exc_info() # kwargs doesn't match a variable in the message # log the issue and the kwargs LOG.exception(_LE('Exception in string format operation')) for name, value in kwargs.items(): LOG.error(_LE("%(name)s: %(value)s"), {'name': name, 'value': value}) if CONF.fatal_exception_format_errors: six.reraise(*exc_info) # at least get the core message out if something happened message = self.message elif isinstance(message, Exception): message = six.text_type(message) # NOTE(luisg): We put the actual message in 'msg' so that we can access # it, because if we try to access the message via 'message' it will be # overshadowed by the class' message attribute self.msg = message super(WaterfallException, self).__init__(message) def _should_format(self): return self.kwargs['message'] is None or '%(message)' in self.message def __unicode__(self): return six.text_type(self.msg)
def get_lock(self, name): """Return a Tooz backend lock. :param str name: The lock name that is used to identify it across all nodes. """ if self.coordinator is not None: return self.coordinator.get_lock(self.prefix + name) else: raise exception.LockCreationFailed(_('Coordinator uninitialized.'))
def matches_versioned_method(self, method): """Compares this version to that of a versioned method.""" if type(method) != versioned_method.VersionedMethod: msg = _('An API version request must be compared ' 'to a VersionedMethod object.') raise exception.InvalidParameterValue(err=msg) return self.matches(method.start_version, method.end_version, method.experimental)
def main(): objects.register_all() gmr_opts.set_defaults(CONF) CONF(sys.argv[1:], project='waterfall', version=version.version_string()) logging.setup(CONF, "waterfall") python_logging.captureWarnings(True) utils.monkey_patch() gmr.TextGuruMeditation.setup_autorun(version, conf=CONF) launcher = service.get_launcher() LOG = logging.getLogger(__name__) service_started = False if CONF.enabled_backends: for backend in CONF.enabled_backends: CONF.register_opt(host_opt, group=backend) backend_host = getattr(CONF, backend).backend_host host = "%s@%s" % (backend_host or CONF.host, backend) try: server = service.Service.create(host=host, service_name=backend, binary='waterfall-workflow') except Exception: msg = _('Workflow service %s failed to start.') % host LOG.exception(msg) else: # Dispose of the whole DB connection pool here before # starting another process. Otherwise we run into cases where # child processes share DB connections which results in errors. session.dispose_engine() launcher.launch_service(server) service_started = True else: server = service.Service.create(binary='waterfall-workflow') launcher.launch_service(server) service_started = True if not service_started: msg = _('No workflow service(s) started successfully, terminating.') LOG.error(msg) sys.exit(1) launcher.wait()
def __init__(self, ip, port, conn_timeout, login, password=None, privatekey=None, *args, **kwargs): self.ip = ip self.port = port self.login = login self.password = password self.conn_timeout = conn_timeout if conn_timeout else None self.privatekey = privatekey self.hosts_key_file = None # Validate good config setting here. # Paramiko handles the case where the file is inaccessible. if not CONF.ssh_hosts_key_file: raise exception.ParameterNotFound(param='ssh_hosts_key_file') elif not os.path.isfile(CONF.ssh_hosts_key_file): # If using the default path, just create the file. if CONF.state_path in CONF.ssh_hosts_key_file: open(CONF.ssh_hosts_key_file, 'a').close() else: msg = (_("Unable to find ssh_hosts_key_file: %s") % CONF.ssh_hosts_key_file) raise exception.InvalidInput(reason=msg) if 'hosts_key_file' in kwargs.keys(): self.hosts_key_file = kwargs.pop('hosts_key_file') LOG.info( _LI("Secondary ssh hosts key file %(kwargs)s will be " "loaded along with %(conf)s from /etc/waterfall.conf."), { 'kwargs': self.hosts_key_file, 'conf': CONF.ssh_hosts_key_file }) LOG.debug( "Setting strict_ssh_host_key_policy to '%(policy)s' " "using ssh_hosts_key_file '%(key_file)s'.", { 'policy': CONF.strict_ssh_host_key_policy, 'key_file': CONF.ssh_hosts_key_file }) self.strict_ssh_host_key_policy = CONF.strict_ssh_host_key_policy if not self.hosts_key_file: self.hosts_key_file = CONF.ssh_hosts_key_file else: self.hosts_key_file += ',' + CONF.ssh_hosts_key_file super(SSHPool, self).__init__(*args, **kwargs)
def get_bool_param(param_string, params): param = params.get(param_string, False) if not is_valid_boolstr(param): msg = _('Value %(param)s for %(param_string)s is not a ' 'boolean.') % { 'param': param, 'param_string': param_string } raise exception.InvalidParameterValue(err=msg) return strutils.bool_from_string(param, strict=True)