def check_string_length(value, name, min_length=0, max_length=None):
"""Check the length of specified string.
:param value: the value of the string
:param name: the name of the string
:param min_length: the min_length of the string
:param max_length: the max_length of the string
"""
if not isinstance(value, six.string_types):
msg = _("%s is not a string or unicode") % name
raise exception.InvalidInput(message=msg)
if len(value) < min_length:
msg = _("%(name)s has a minimum character requirement of "
"%(min_length)s.") % {
'name': name,
'min_length': min_length
}
raise exception.InvalidInput(message=msg)
if max_length and len(value) > max_length:
msg = _("%(name)s has more than %(max_length)s "
"characters.") % {
'name': name,
'max_length': max_length
}
raise exception.InvalidInput(message=msg)
def validate_integer(value, name, min_value=None, max_value=None):
"""Make sure that value is a valid integer, potentially within range.
:param value: the value of the integer
:param name: the name of the integer
:param min_length: the min_length of the integer
:param max_length: the max_length of the integer
:returns: integer
"""
try:
value = int(value)
except (TypeError, ValueError, UnicodeEncodeError):
raise webob.exc.HTTPBadRequest(
explanation=(_('%s must be an integer.') % name))
if min_value is not None and value < min_value:
raise webob.exc.HTTPBadRequest(
explanation=(_('%(value_name)s must be >= %(min_value)d') % {
'value_name': name,
'min_value': min_value
}))
if max_value is not None and value > max_value:
raise webob.exc.HTTPBadRequest(
explanation=(_('%(value_name)s must be <= %(max_value)d') % {
'value_name': name,
'max_value': max_value
}))
return value
def __init__(self, message=None, **kwargs):
if kwargs.get('host', None):
self.message = _("Service %(service_id)s could not be "
"found on host %(host)s.")
else:
self.message = _("Service %(service_id)s could not be found.")
super(ServiceNotFound, self).__init__(None, **kwargs)
def save(self):
updates = self.waterfall_obj_get_changes()
if updates:
if 'consistencygroup' in updates:
raise exception.ObjectActionError(
action='save', reason=_('consistencygroup changed'))
if 'glance_metadata' in updates:
raise exception.ObjectActionError(
action='save', reason=_('glance_metadata changed'))
if 'snapshots' in updates:
raise exception.ObjectActionError(
action='save', reason=_('snapshots changed'))
if 'metadata' in updates:
# Metadata items that are not specified in the
# self.metadata will be deleted
metadata = updates.pop('metadata', None)
self.metadata = db.workflow_metadata_update(
self._context, self.id, metadata, True)
if self._context.is_admin and 'admin_metadata' in updates:
metadata = updates.pop('admin_metadata', None)
self.admin_metadata = db.workflow_admin_metadata_update(
self._context, self.id, metadata, True)
db.workflow_update(self._context, self.id, updates)
self.obj_reset_changes()
def save(self):
updates = self.waterfall_obj_get_changes()
if updates:
if 'consistencygroup' in updates:
raise exception.ObjectActionError(
action='save', reason=_('consistencygroup changed'))
if 'snapshots' in updates:
raise exception.ObjectActionError(
action='save', reason=_('snapshots changed'))
db.cgsnapshot_update(self._context, self.id, updates)
self.obj_reset_changes()
def create(self):
try:
ssh = paramiko.SSHClient()
if ',' in self.hosts_key_file:
files = self.hosts_key_file.split(',')
for f in files:
ssh.load_host_keys(f)
else:
ssh.load_host_keys(self.hosts_key_file)
# If strict_ssh_host_key_policy is set we want to reject, by
# default if there is not entry in the known_hosts file.
# Otherwise we use AutoAddPolicy which accepts on the first
# Connect but fails if the keys change. load_host_keys can
# handle hashed known_host entries.
if self.strict_ssh_host_key_policy:
ssh.set_missing_host_key_policy(paramiko.RejectPolicy())
else:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if self.password:
ssh.connect(self.ip,
port=self.port,
username=self.login,
password=self.password,
timeout=self.conn_timeout)
elif self.privatekey:
pkfile = os.path.expanduser(self.privatekey)
privatekey = paramiko.RSAKey.from_private_key_file(pkfile)
ssh.connect(self.ip,
port=self.port,
username=self.login,
pkey=privatekey,
timeout=self.conn_timeout)
else:
msg = _("Specify a password or private_key")
raise exception.WaterfallException(msg)
# Paramiko by default sets the socket timeout to 0.1 seconds,
# ignoring what we set through the sshclient. This doesn't help for
# keeping long lived connections. Hence we have to bypass it, by
# overriding it after the transport is initialized. We are setting
# the sockettimeout to None and setting a keepalive packet so that,
# the server will keep the connection open. All that does is send
# a keepalive packet every ssh_conn_timeout seconds.
if self.conn_timeout:
transport = ssh.get_transport()
transport.sock.settimeout(None)
transport.set_keepalive(self.conn_timeout)
return ssh
except Exception as e:
msg = _("Error connecting via ssh: %s") % six.text_type(e)
LOG.error(msg)
raise paramiko.SSHException(msg)
def create(self):
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason=_('already_created'))
updates = self.waterfall_obj_get_changes()
if 'consistencygroup' in updates:
raise exception.ObjectActionError(
action='create', reason=_('consistencygroup assigned'))
db_cgsnapshots = db.cgsnapshot_create(self._context, updates)
self._from_db_object(self._context, self, db_cgsnapshots)
def decode_record(backup_url):
"""Deserialize backup metadata from string into a dictionary.
:raises: InvalidInput
"""
try:
return jsonutils.loads(base64.decode_as_text(backup_url))
except TypeError:
msg = _("Can't decode backup record.")
except ValueError:
msg = _("Can't parse backup record.")
raise exception.InvalidInput(reason=msg)
def create(self):
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason=_('already created'))
updates = self.waterfall_obj_get_changes()
if 'workflow' in updates:
raise exception.ObjectActionError(action='create',
reason=_('workflow assigned'))
if 'cgsnapshot' in updates:
raise exception.ObjectActionError(action='create',
reason=_('cgsnapshot assigned'))
db_snapshot = db.snapshot_create(self._context, updates)
self._from_db_object(self._context, self, db_snapshot)
class WorkflowLimitExceeded(QuotaError):
message = _("Maximum number of workflows allowed (%(allowed)d) exceeded for "
"quota '%(name)s'.")
def __init__(self, message=None, **kwargs):
kwargs.setdefault('name', 'workflows')
super(WorkflowLimitExceeded, self).__init__(message, **kwargs)
class APIException(WaterfallException):
message = _("Error while requesting %(service)s API.")
def __init__(self, message=None, **kwargs):
if 'service' not in kwargs:
kwargs['service'] = 'unknown'
super(APIException, self).__init__(message, **kwargs)
def __init__(self, name, loader=None):
"""Initialize, but do not start the WSGI server.
:param name: The name of the WSGI server given to the loader.
:param loader: Loads the WSGI application using the given name.
:returns: None
"""
self.name = name
self.manager = self._get_manager()
self.loader = loader or wsgi.Loader(CONF)
self.app = self.loader.load_app(name)
self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
self.port = getattr(CONF, '%s_listen_port' % name, 0)
self.workers = (getattr(CONF, '%s_workers' % name, None)
or processutils.get_worker_count())
if self.workers and self.workers < 1:
worker_name = '%s_workers' % name
msg = (_("%(worker_name)s value of %(workers)d is invalid, "
"must be greater than 0.") % {
'worker_name': worker_name,
'workers': self.workers
})
raise exception.InvalidInput(msg)
setup_profiler(name, self.host)
self.server = wsgi.Server(CONF,
name,
self.app,
host=self.host,
port=self.port)
def _db_error(caught_exception):
print('%s' % caught_exception)
print(
_("The above error may show that the database has not "
"been created.\nPlease create a database using "
"'waterfall-manage db sync' before running this command."))
sys.exit(1)
def _error(self, inner, req):
if not isinstance(inner, exception.QuotaError):
LOG.exception(_LE("Caught error: %(type)s %(error)s"), {
'type': type(inner),
'error': inner
})
safe = getattr(inner, 'safe', False)
headers = getattr(inner, 'headers', None)
status = getattr(inner, 'code', 500)
if status is None:
status = 500
msg_dict = dict(url=req.url, status=status)
LOG.info(_LI("%(url)s returned with HTTP %(status)d"), msg_dict)
outer = self.status_to_type(status)
if headers:
outer.headers = headers
# NOTE(johannes): We leave the explanation empty here on
# purpose. It could possibly have sensitive information
# that should not be returned back to the user. See
# bugs 868360 and 874472
# NOTE(eglynn): However, it would be over-conservative and
# inconsistent with the EC2 API to hide every exception,
# including those that are safe to expose, see bug 1021373
if safe:
msg = (inner.msg if isinstance(inner, exception.WaterfallException)
else six.text_type(inner))
params = {
'exception': inner.__class__.__name__,
'explanation': msg
}
outer.explanation = _('%(exception)s: %(explanation)s') % params
return wsgi.Fault(outer)
def __call__(self, environ, start_response):
r"""Subclasses will probably want to implement __call__ like this:
@webob.dec.wsgify(RequestClass=Request)
def __call__(self, req):
# Any of the following objects work as responses:
# Option 1: simple string
res = 'message\n'
# Option 2: a nicely formatted HTTP exception page
res = exc.HTTPForbidden(explanation='Nice try')
# Option 3: a webob Response object (in case you need to play with
# headers, or you want to be treated like an iterable)
res = Response();
res.app_iter = open('somefile')
# Option 4: any wsgi app to be run next
res = self.application
# Option 5: you can get a Response object for a wsgi app, too, to
# play with headers etc
res = req.get_response(self.application)
# You can then just return your response...
return res
# ... or set req.response and return None.
req.response = res
See the end of http://pythonpaste.org/webob/modules/dec.html
for more info.
"""
raise NotImplementedError(_('You must implement __call__'))
def create(self):
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason=_('already created'))
updates = self.waterfall_obj_get_changes()
db_service = db.service_create(self._context, updates)
self._from_db_object(self._context, self, db_service)
def check_exclusive_options(**kwargs):
"""Checks that only one of the provided options is actually not-none.
Iterates over all the kwargs passed in and checks that only one of said
arguments is not-none, if more than one is not-none then an exception will
be raised with the names of those arguments who were not-none.
"""
if not kwargs:
return
pretty_keys = kwargs.pop("pretty_keys", True)
exclusive_options = {}
for (k, v) in kwargs.items():
if v is not None:
exclusive_options[k] = True
if len(exclusive_options) > 1:
# Change the format of the names from pythonic to
# something that is more readable.
#
# Ex: 'the_key' -> 'the key'
if pretty_keys:
names = [k.replace('_', ' ') for k in kwargs.keys()]
else:
names = kwargs.keys()
names = ", ".join(sorted(names))
msg = (_("May specify only one of %s") % (names))
raise exception.InvalidInput(reason=msg)
def attach(self, *slaves):
"""Attach one or more slave templates.
Attaches one or more slave templates to the master template.
Slave templates must have a root element with the same tag as
the master template. The slave template's apply() method will
be called to determine if the slave should be applied to this
master; if it returns False, that slave will be skipped.
(This allows filtering of slaves based on the version of the
master template.)
"""
slave_list = []
for slave in slaves:
slave = slave.wrap()
# Make sure we have a tree match
if slave.root.tag != self.root.tag:
msg = (_("Template tree mismatch; adding slave %(slavetag)s "
"to master %(mastertag)s") % {
'slavetag': slave.root.tag,
'mastertag': self.root.tag
})
raise ValueError(msg)
# Make sure slave applies to this template
if not slave.apply(self):
continue
slave_list.append(slave)
# Add the slaves
self.slaves.extend(slave_list)
def get_blkdev_major_minor(path, lookup_for_file=True):
"""Get 'major:minor' number of block device.
Get the device's 'major:minor' number of a block device to control
I/O ratelimit of the specified path.
If lookup_for_file is True and the path is a regular file, lookup a disk
device which the file lies on and returns the result for the device.
"""
st = os.stat(path)
if stat.S_ISBLK(st.st_mode):
path, st = _get_disk_of_partition(path, st)
return '%d:%d' % (os.major(st.st_rdev), os.minor(st.st_rdev))
elif stat.S_ISCHR(st.st_mode):
# No I/O ratelimit control is provided for character devices
return None
elif lookup_for_file:
# lookup the mounted disk which the file lies on
out, _err = execute('df', path)
devpath = out.split("\n")[1].split()[0]
if devpath[0] is not '/':
# the file is on a network file system
return None
return get_blkdev_major_minor(devpath, False)
else:
msg = _("Unable to get a block device for file \'%s\'") % path
raise exception.Error(msg)
def _get_limit_param(params, max_limit=None):
"""Extract integer limit from request's dictionary or fail.
Defaults to max_limit if not present and returns max_limit if present
'limit' is greater than max_limit.
"""
max_limit = max_limit or CONF.osapi_max_limit
try:
limit = int(params.pop('limit', max_limit))
except ValueError:
msg = _('limit param must be an integer')
raise webob.exc.HTTPBadRequest(explanation=msg)
if limit < 0:
msg = _('limit param must be positive')
raise webob.exc.HTTPBadRequest(explanation=msg)
limit = min(limit, max_limit)
return limit
def purge(self, age_in_days):
"""Purge deleted rows older than a given age from waterfall tables."""
age_in_days = int(age_in_days)
if age_in_days <= 0:
print(_("Must supply a positive, non-zero value for age"))
sys.exit(1)
ctxt = context.get_admin_context()
db.purge_deleted_rows(ctxt, age_in_days)
def construct(self):
"""Construct a template.
Called to construct a template instance, which it must return.
Only called once.
"""
raise NotImplementedError(_("subclasses must implement construct()!"))
def create(self):
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason=_('already created'))
db_workflow_type = workflow_types.create(self._context, self.name,
self.extra_specs,
self.is_public, self.projects,
self.description)
self._from_db_object(self._context, self, db_workflow_type)
class WorkflowSizeExceedsAvailableQuota(QuotaError):
message = _("Requested workflow or snapshot exceeds allowed %(name)s "
"quota. Requested %(requested)sG, quota is %(quota)sG and "
"%(consumed)sG has been consumed.")
def __init__(self, message=None, **kwargs):
kwargs.setdefault('name', 'gigabytes')
super(WorkflowSizeExceedsAvailableQuota, self).__init__(
message, **kwargs)
class WaterfallException(Exception):
"""Base Waterfall Exception
To correctly use this class, inherit from it and define
a 'message' property. That message will get printf'd
with the keyword arguments provided to the constructor.
"""
message = _("An unknown exception occurred.")
code = 500
headers = {}
safe = False
def __init__(self, message=None, **kwargs):
self.kwargs = kwargs
self.kwargs['message'] = message
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
for k, v in self.kwargs.items():
if isinstance(v, Exception):
self.kwargs[k] = six.text_type(v)
if self._should_format():
try:
message = self.message % kwargs
except Exception:
exc_info = sys.exc_info()
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception(_LE('Exception in string format operation'))
for name, value in kwargs.items():
LOG.error(_LE("%(name)s: %(value)s"),
{'name': name, 'value': value})
if CONF.fatal_exception_format_errors:
six.reraise(*exc_info)
# at least get the core message out if something happened
message = self.message
elif isinstance(message, Exception):
message = six.text_type(message)
# NOTE(luisg): We put the actual message in 'msg' so that we can access
# it, because if we try to access the message via 'message' it will be
# overshadowed by the class' message attribute
self.msg = message
super(WaterfallException, self).__init__(message)
def _should_format(self):
return self.kwargs['message'] is None or '%(message)' in self.message
def __unicode__(self):
return six.text_type(self.msg)
def get_lock(self, name):
"""Return a Tooz backend lock.
:param str name: The lock name that is used to identify it
across all nodes.
"""
if self.coordinator is not None:
return self.coordinator.get_lock(self.prefix + name)
else:
raise exception.LockCreationFailed(_('Coordinator uninitialized.'))
def matches_versioned_method(self, method):
"""Compares this version to that of a versioned method."""
if type(method) != versioned_method.VersionedMethod:
msg = _('An API version request must be compared '
'to a VersionedMethod object.')
raise exception.InvalidParameterValue(err=msg)
return self.matches(method.start_version, method.end_version,
method.experimental)
def main():
objects.register_all()
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project='waterfall', version=version.version_string())
logging.setup(CONF, "waterfall")
python_logging.captureWarnings(True)
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
launcher = service.get_launcher()
LOG = logging.getLogger(__name__)
service_started = False
if CONF.enabled_backends:
for backend in CONF.enabled_backends:
CONF.register_opt(host_opt, group=backend)
backend_host = getattr(CONF, backend).backend_host
host = "%s@%s" % (backend_host or CONF.host, backend)
try:
server = service.Service.create(host=host,
service_name=backend,
binary='waterfall-workflow')
except Exception:
msg = _('Workflow service %s failed to start.') % host
LOG.exception(msg)
else:
# Dispose of the whole DB connection pool here before
# starting another process. Otherwise we run into cases where
# child processes share DB connections which results in errors.
session.dispose_engine()
launcher.launch_service(server)
service_started = True
else:
server = service.Service.create(binary='waterfall-workflow')
launcher.launch_service(server)
service_started = True
if not service_started:
msg = _('No workflow service(s) started successfully, terminating.')
LOG.error(msg)
sys.exit(1)
launcher.wait()
def __init__(self,
ip,
port,
conn_timeout,
login,
password=None,
privatekey=None,
*args,
**kwargs):
self.ip = ip
self.port = port
self.login = login
self.password = password
self.conn_timeout = conn_timeout if conn_timeout else None
self.privatekey = privatekey
self.hosts_key_file = None
# Validate good config setting here.
# Paramiko handles the case where the file is inaccessible.
if not CONF.ssh_hosts_key_file:
raise exception.ParameterNotFound(param='ssh_hosts_key_file')
elif not os.path.isfile(CONF.ssh_hosts_key_file):
# If using the default path, just create the file.
if CONF.state_path in CONF.ssh_hosts_key_file:
open(CONF.ssh_hosts_key_file, 'a').close()
else:
msg = (_("Unable to find ssh_hosts_key_file: %s") %
CONF.ssh_hosts_key_file)
raise exception.InvalidInput(reason=msg)
if 'hosts_key_file' in kwargs.keys():
self.hosts_key_file = kwargs.pop('hosts_key_file')
LOG.info(
_LI("Secondary ssh hosts key file %(kwargs)s will be "
"loaded along with %(conf)s from /etc/waterfall.conf."), {
'kwargs': self.hosts_key_file,
'conf': CONF.ssh_hosts_key_file
})
LOG.debug(
"Setting strict_ssh_host_key_policy to '%(policy)s' "
"using ssh_hosts_key_file '%(key_file)s'.", {
'policy': CONF.strict_ssh_host_key_policy,
'key_file': CONF.ssh_hosts_key_file
})
self.strict_ssh_host_key_policy = CONF.strict_ssh_host_key_policy
if not self.hosts_key_file:
self.hosts_key_file = CONF.ssh_hosts_key_file
else:
self.hosts_key_file += ',' + CONF.ssh_hosts_key_file
super(SSHPool, self).__init__(*args, **kwargs)
def get_bool_param(param_string, params):
param = params.get(param_string, False)
if not is_valid_boolstr(param):
msg = _('Value %(param)s for %(param_string)s is not a '
'boolean.') % {
'param': param,
'param_string': param_string
}
raise exception.InvalidParameterValue(err=msg)
return strutils.bool_from_string(param, strict=True)