def limited(items, request, max_limit=CONF.osapi_max_limit):
"""Return a slice of items according to requested offset and limit.
:param items: A sliceable entity
:param request: ``wsgi.Request`` possibly containing 'offset' and 'limit'
GET variables. 'offset' is where to start in the list,
and 'limit' is the maximum number of items to return. If
'limit' is not specified, 0, or > max_limit, we default
to max_limit. Negative values for either offset or limit
will cause exc.HTTPBadRequest() exceptions to be raised.
:kwarg max_limit: The maximum number of items to return from 'items'
"""
try:
offset = int(request.GET.get('offset', 0))
except ValueError:
msg = _('offset param must be an integer')
raise webob.exc.HTTPBadRequest(explanation=msg)
try:
limit = int(request.GET.get('limit', max_limit))
except ValueError:
msg = _('limit param must be an integer')
raise webob.exc.HTTPBadRequest(explanation=msg)
if limit < 0:
msg = _('limit param must be positive')
raise webob.exc.HTTPBadRequest(explanation=msg)
if offset < 0:
msg = _('offset param must be positive')
raise webob.exc.HTTPBadRequest(explanation=msg)
limit = min(max_limit, limit or max_limit)
range_end = offset + limit
return items[offset:range_end]
def wait_for_access_update(context, db, learning_instance,
migration_wait_access_rules_timeout):
starttime = time.time()
deadline = starttime + migration_wait_access_rules_timeout
tries = 0
while True:
instance = db.learning_instance_get(context, learning_instance['id'])
if instance['access_rules_status'] == constants.STATUS_ACTIVE:
break
tries += 1
now = time.time()
if instance['access_rules_status'] == constants.STATUS_ERROR:
msg = _("Failed to update access rules"
" on learning instance %s") % learning_instance['id']
raise exception.LearningMigrationFailed(reason=msg)
elif now > deadline:
msg = _("Timeout trying to update access rules"
" on learning instance %(learning_id)s. Timeout "
"was %(timeout)s seconds.") % {
'learning_id': learning_instance['id'],
'timeout': migration_wait_access_rules_timeout
}
raise exception.LearningMigrationFailed(reason=msg)
else:
time.sleep(tries**2)
def main():
"""Parse options and call the appropriate class/method."""
CONF.register_cli_opt(category_opt)
script_name = sys.argv[0]
if len(sys.argv) < 2:
print(
_("\nOpenStack meteos version: %(version)s\n") %
{'version': version.version_string()})
print(script_name + " category action [<args>]")
print(_("Available categories:"))
for category in CATEGORIES:
print("\t%s" % category)
sys.exit(2)
try:
log.register_options(CONF)
CONF(sys.argv[1:], project='meteos', version=version.version_string())
log.setup(CONF, "meteos")
except cfg.ConfigFilesNotFoundError:
cfgfile = CONF.config_file[-1] if CONF.config_file else None
if cfgfile and not os.access(cfgfile, os.R_OK):
st = os.stat(cfgfile)
print(_("Could not read %s. Re-running with sudo") % cfgfile)
try:
os.execvp('sudo', ['sudo', '-u', '#%s' % st.st_uid] + sys.argv)
except Exception:
print(_('sudo failed, continuing as if nothing happened'))
print(_('Please re-run meteos-manage as root.'))
sys.exit(2)
fn = CONF.category.action_fn
fn_args = fetch_func_args(fn)
fn(*fn_args)
def _get_limit_param(request):
"""Extract integer limit from request or fail."""
try:
limit = int(request.GET['limit'])
except ValueError:
msg = _('limit param must be an integer')
raise webob.exc.HTTPBadRequest(explanation=msg)
if limit < 0:
msg = _('limit param must be positive')
raise webob.exc.HTTPBadRequest(explanation=msg)
return limit
def validate_update(self, body, status_attr='status'):
update = {}
try:
update[status_attr] = body[status_attr]
except (TypeError, KeyError):
msg = _("Must specify '%s'") % status_attr
raise webob.exc.HTTPBadRequest(explanation=msg)
if update[status_attr] not in self.valid_statuses[status_attr]:
expl = (_("Invalid state. Valid states: %s.") % ", ".join(
six.text_type(i) for i in self.valid_statuses[status_attr]))
raise webob.exc.HTTPBadRequest(explanation=expl)
return update
def is_admin_context(context):
"""Indicates if the request context is an administrator."""
if not context:
warnings.warn(_('Use of empty request context is deprecated'),
DeprecationWarning)
raise Exception('die')
return context.is_admin
def __call__(self, environ, start_response):
r"""Subclasses will probably want to implement __call__ like this:
@webob.dec.wsgify(RequestClass=Request)
def __call__(self, req):
# Any of the following objects work as responses:
# Option 1: simple string
res = 'message\n'
# Option 2: a nicely formatted HTTP exception page
res = exc.HTTPForbidden(detail='Nice try')
# Option 3: a webob Response object (in case you need to play with
# headers, or you want to be treated like an iterable, or or or)
res = Response();
res.app_iter = open('somefile')
# Option 4: any wsgi app to be run next
res = self.application
# Option 5: you can get a Response object for a wsgi app, too, to
# play with headers etc
res = req.get_response(self.application)
# You can then just return your response...
return res
# ... or set req.response and return None.
req.response = res
See the end of http://pythonpaste.org/webob/modules/dec.html
for more info.
"""
raise NotImplementedError(_('You must implement __call__'))
def load_model(self, context, id, dataset_format, model_type,
job_template_id, experiment_id, cluster_id):
"""Load a Model"""
policy.check_policy(context, 'model', 'load')
model = {
'id': id,
'dataset_format': dataset_format,
'model_type': model_type,
'job_template_id': job_template_id,
'experiment_id': experiment_id,
'cluster_id': cluster_id
}
if not self._enable_load_model(context):
msg = _("Can not load multiple models at the same time.")
raise exception.InvalidLearning(reason=msg)
try:
self.engine_rpcapi.load_model(context, model)
updates = {'status': constants.STATUS_ACTIVATING}
LOG.info("Accepted load of model %s.", id)
self.db.model_update(context, id, updates)
except Exception:
with excutils.save_and_reraise_exception():
updates = {'status': constants.STATUS_ERROR}
self.db.model_update(context, id, updates)
def is_valid_status(resource_name, status, valid_statuses):
if status not in valid_statuses:
msg = _("%(resource_name)s status must be %(valid_statuses)s") % {
"resource_name": resource_name,
"valid_statuses": valid_statuses
}
raise exception.InvalidStatus(reason=msg)
def action_peek_json(body):
"""Determine action to invoke."""
try:
decoded = jsonutils.loads(body)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
# Make sure there's exactly one key...
if len(decoded) != 1:
msg = _("too many body keys")
raise exception.MalformedRequestBody(reason=msg)
# Return the action and the decoded body...
return list(decoded.keys())[0]
def create(self):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
look_for_keys = True
if self.path_to_private_key:
self.path_to_private_key = os.path.expanduser(
self.path_to_private_key)
look_for_keys = False
elif self.password:
look_for_keys = False
try:
ssh.connect(self.ip,
port=self.port,
username=self.login,
password=self.password,
key_filename=self.path_to_private_key,
look_for_keys=look_for_keys,
timeout=self.conn_timeout)
# Paramiko by default sets the socket timeout to 0.1 seconds,
# ignoring what we set through the sshclient. This doesn't help for
# keeping long lived connections. Hence, we have to bypass it, by
# overriding it after the transport is initialized. We are setting
# the sockettimeout to None and setting a keepalive packet so that,
# the server will keep the connection open. All that does is sent
# a keepalive packet every ssh_conn_timeout seconds.
if self.conn_timeout:
transport = ssh.get_transport()
transport.sock.settimeout(None)
transport.set_keepalive(self.conn_timeout)
return ssh
except Exception as e:
msg = _("Check whether private key or password are correctly "
"set. Error connecting via ssh: %s") % e
LOG.error(msg)
raise exception.SSHException(msg)
def _wait_for_model_to_load(self, ip, port, unload=False):
stdout = ""
stderr = ""
starttime = time.time()
deadline = starttime + self.configuration.load_model_timeout
interval = self.configuration.api_retry_interval
tries = 0
while True:
try:
stdout, stderr = self._run_ssh(
ip, ['netstat', '-tnl', '|', 'grep', port])
except processutils.ProcessExecutionError:
pass
if not stdout and unload:
break
if stdout and not unload:
break
tries += 1
now = time.time()
if now > deadline:
msg = _("Timeout trying to load/unload model " "%s") % id
raise exception.Invalid(reason=msg)
LOG.debug("Waiting for load/unload to complete")
time.sleep(interval)
def get_job_result(self, context, job_id, template_id, cluster_id):
stdout = ""
stderr = ""
starttime = time.time()
deadline = starttime + self.configuration.execute_job_timeout
interval = self.configuration.api_retry_interval
tries = 0
while True:
job = self.cluster_api.get_job(context, job_id)
if job.info['status'] == const.STATUS_JOB_SUCCESS:
stdout = self._get_job_result(context, template_id, cluster_id,
job_id)
break
elif job.info['status'] == const.STATUS_JOB_ERROR:
stderr = self._get_job_result(context, template_id, cluster_id,
job_id)
break
tries += 1
now = time.time()
if now > deadline:
msg = _("Timeout trying to create experiment " "%s") % job_id
raise exception.Invalid(reason=msg)
LOG.debug("Waiting for job to complete: Current status: %s",
job.info['status'])
time.sleep(interval)
return stdout, stderr
def list(self, zone=None):
"""Show a list of all physical hosts. Filter by zone.
args: [zone]
"""
print("%-25s\t%-15s" % (_('host'), _('zone')))
ctxt = context.get_admin_context()
services = db.service_get_all(ctxt)
if zone:
services = [
s for s in services if s['availability_zone']['name'] == zone
]
hosts = []
for srv in services:
if not [h for h in hosts if h['host'] == srv['host']]:
hosts.append(srv)
for h in hosts:
print("%-25s\t%-15s" % (h['host'], h['availability_zone']['name']))
def matches_versioned_method(self, method):
"""Compares this version to that of a versioned method."""
if type(method) != versioned_method.VersionedMethod:
msg = _('An API version request must be compared '
'to a VersionedMethod object.')
raise exception.InvalidParameterValue(err=msg)
return self.matches(method.start_version,
method.end_version,
method.experimental)
def _model_evaluation_get_all_with_filters(context,
project_id=None,
filters=None,
sort_key=None,
sort_dir=None):
if not sort_key:
sort_key = 'created_at'
if not sort_dir:
sort_dir = 'desc'
query = (_model_evaluation_get_query(context).join())
query = query.filter(models.Model_Evaluation.project_id == project_id)
# Apply filters
if not filters:
filters = {}
# Apply sorting
if sort_dir.lower() not in ('desc', 'asc'):
msg = _("Wrong sorting data provided: sort key is '%(sort_key)s' "
"and sort direction is '%(sort_dir)s'.") % {
"sort_key": sort_key,
"sort_dir": sort_dir
}
raise exception.InvalidInput(reason=msg)
def apply_sorting(model_evaluation, query):
sort_attr = getattr(model_evaluation, sort_key)
sort_method = getattr(sort_attr, sort_dir.lower())
return query.order_by(sort_method())
try:
query = apply_sorting(models.Model_Evaluation, query)
except AttributeError:
msg = _("Wrong sorting key provided - '%s'.") % sort_key
raise exception.InvalidInput(reason=msg)
# Returns list of model_evaluations that satisfy filters.
query = query.all()
return query
class MeteosException(Exception):
"""Base Meteos Exception
To correctly use this class, inherit from it and define
a 'message' property. That message will get printf'd
with the keyword arguments provided to the constructor.
"""
message = _("An unknown exception occurred.")
code = 500
headers = {}
safe = False
def __init__(self, message=None, detail_data={}, **kwargs):
self.kwargs = kwargs
self.detail_data = detail_data
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
for k, v in self.kwargs.items():
if isinstance(v, Exception):
self.kwargs[k] = six.text_type(v)
if not message:
try:
message = self.message % kwargs
except Exception:
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception('Exception in string format operation.')
for name, value in kwargs.items():
LOG.error("%(name)s: %(value)s", {
'name': name,
'value': value
})
if CONF.fatal_exception_format_errors:
raise
else:
# at least get the core message out if something happened
message = self.message
elif isinstance(message, Exception):
message = six.text_type(message)
if re.match('.*[^\.]\.\.$', message):
message = message[:-1]
self.msg = message
super(MeteosException, self).__init__(message)
def _load_auth_plugin(self):
if self.admin_auth:
return self.admin_auth
self.auth_plugin = ks_loading.load_auth_from_conf_options(
CONF, self.group)
if self.url.find('v2') > -1:
self.auth_plugin = v2.Token().load_from_options(
**self.deprecated_opts_for_v2)
else:
self.auth_plugin = v3.Token().load_from_options(**self.opts_for_v3)
if self.auth_plugin:
return self.auth_plugin
msg = _('Cannot load auth plugin for %s') % self.group
raise self.exception_module.Unauthorized(message=msg)
def __get_backend(self):
if not self.__backend:
backend_name = CONF[self.__pivot]
if backend_name not in self.__backends:
raise exception.Error(_('Invalid backend: %s') % backend_name)
backend = self.__backends[backend_name]
if isinstance(backend, tuple):
name = backend[0]
fromlist = backend[1]
else:
name = backend
fromlist = backend
self.__backend = __import__(name, None, None, fromlist)
LOG.debug('backend %s', self.__backend)
return self.__backend
def __call__(self, req):
user_id = req.headers.get('X_USER')
user_id = req.headers.get('X_USER_ID', user_id)
if user_id is None:
LOG.debug("Neither X_USER_ID nor X_USER found in request")
return webob.exc.HTTPUnauthorized()
# get the roles
roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')]
if 'X_TENANT_ID' in req.headers:
# This is the new header since Keystone went to ID/Name
project_id = req.headers['X_TENANT_ID']
else:
# This is for legacy compatibility
project_id = req.headers['X_TENANT']
# Get the auth token
auth_token = req.headers.get('X_AUTH_TOKEN',
req.headers.get('X_STORAGE_TOKEN'))
# Build a context, including the auth_token...
remote_address = req.remote_addr
if CONF.use_forwarded_for:
remote_address = req.headers.get('X-Forwarded-For', remote_address)
service_catalog = None
if req.headers.get('X_SERVICE_CATALOG') is not None:
try:
catalog_header = req.headers.get('X_SERVICE_CATALOG')
service_catalog = jsonutils.loads(catalog_header)
except ValueError:
raise webob.exc.HTTPInternalServerError(
_('Invalid service catalog json.'))
ctx = context.RequestContext(user_id,
project_id,
roles=roles,
auth_token=auth_token,
remote_address=remote_address,
service_catalog=service_catalog)
req.environ['meteos.context'] = ctx
return self.application
def wait_for_cluster_create(self, context, id):
starttime = time.time()
deadline = starttime + self.configuration.create_experiment_timeout
interval = self.configuration.api_retry_interval
tries = 0
while True:
cluster = self.cluster_api.get_cluster(context, id)
if cluster.status == const.STATUS_SAHARA_ACTIVE:
break
tries += 1
now = time.time()
if now > deadline:
msg = _("Timeout trying to create experiment " "%s") % id
raise exception.Invalid(reason=msg)
LOG.debug("Waiting for cluster to complete: Current status: %s",
cluster.status)
time.sleep(interval)
def limited_by_marker(items, request, max_limit=CONF.osapi_max_limit):
"""Return a slice of items according to the requested marker and limit."""
params = get_pagination_params(request)
limit = params.get('limit', max_limit)
marker = params.get('marker')
limit = min(max_limit, limit)
start_index = 0
if marker:
start_index = -1
for i, item in enumerate(items):
if 'flavorid' in item:
if item['flavorid'] == marker:
start_index = i + 1
break
elif item['id'] == marker or item.get('uuid') == marker:
start_index = i + 1
break
if start_index < 0:
msg = _('marker [%s] not found') % marker
raise webob.exc.HTTPBadRequest(explanation=msg)
range_end = start_index + limit
return items[start_index:range_end]
def list(self):
"""Show a list of all meteos services."""
ctxt = context.get_admin_context()
services = db.service_get_all(ctxt)
print_format = "%-16s %-36s %-16s %-10s %-5s %-10s"
print(print_format % (_('Binary'), _('Host'), _('Zone'), _('Status'),
_('State'), _('Updated At')))
for svc in services:
alive = utils.service_is_up(svc)
art = ":-)" if alive else "XXX"
status = 'enabled'
if svc['disabled']:
status = 'disabled'
print(print_format % (
svc['binary'],
svc['host'].partition('.')[0],
svc['availability_zone']['name'],
status,
art,
svc['updated_at'],
))
class Duplicated(MeteosException):
message = _("Duplicate entry")
code = 409
safe = True
def experimental(self, value):
if type(value) != bool:
msg = _('The experimental property must be a bool value.')
raise exception.InvalidParameterValue(err=msg)
self._experimental = value
def _set_read_deleted(self, read_deleted):
if read_deleted not in ('no', 'yes', 'only'):
raise ValueError(
_("read_deleted can only be one of 'no', "
"'yes' or 'only', not %r") % read_deleted)
self._read_deleted = read_deleted
def serve(server, workers=None):
global _launcher
if _launcher:
raise RuntimeError(_('serve() can only be called once'))
_launcher = service.launch(CONF, server, workers=workers)
def _from_json(self, datastring):
try:
return jsonutils.loads(datastring)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
def _process_stack(self, request, action, action_args, content_type, body,
accept):
"""Implement the processing stack."""
# Get the implementing method
try:
meth, extensions = self.get_method(request, action, content_type,
body)
except (AttributeError, TypeError):
return Fault(webob.exc.HTTPNotFound())
except KeyError as ex:
msg = _("There is no such action: %s") % ex.args[0]
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
except exception.MalformedRequestBody:
msg = _("Malformed request body")
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
if body:
msg = ("Action: '%(action)s', calling method: %(meth)s, body: "
"%(body)s") % {
'action': action,
'body': six.text_type(body),
'meth': six.text_type(meth)
}
LOG.debug(strutils.mask_password(msg))
else:
LOG.debug("Calling method '%(meth)s'",
{'meth': six.text_type(meth)})
# Now, deserialize the request body...
try:
if content_type:
contents = self.deserialize(meth, content_type, body)
else:
contents = {}
except exception.InvalidContentType:
msg = _("Unsupported Content-Type")
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
except exception.MalformedRequestBody:
msg = _("Malformed request body")
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
# Update the action args
action_args.update(contents)
project_id = action_args.pop("project_id", None)
context = request.environ.get('meteos.context')
if (context and project_id and (project_id != context.project_id)):
msg = _("Malformed request url")
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
# Run pre-processing extensions
response, post = self.pre_process_extensions(extensions, request,
action_args)
if not response:
try:
with ResourceExceptionHandler():
action_result = self.dispatch(meth, request, action_args)
except Fault as ex:
response = ex
if not response:
# No exceptions; convert action_result into a
# ResponseObject
resp_obj = None
if type(action_result) is dict or action_result is None:
resp_obj = ResponseObject(action_result)
elif isinstance(action_result, ResponseObject):
resp_obj = action_result
else:
response = action_result
# Run post-processing extensions
if resp_obj:
_set_request_id_header(request, resp_obj)
# Do a preserialize to set up the response object
serializers = getattr(meth, 'wsgi_serializers', {})
resp_obj._bind_method_serializers(serializers)
if hasattr(meth, 'wsgi_code'):
resp_obj._default_code = meth.wsgi_code
resp_obj.preserialize(accept, self.default_serializers)
# Process post-processing extensions
response = self.post_process_extensions(
post, resp_obj, request, action_args)
if resp_obj and not response:
response = resp_obj.serialize(request, accept,
self.default_serializers)
try:
msg_dict = dict(url=request.url, status=response.status_int)
msg = _("%(url)s returned with HTTP %(status)d") % msg_dict
except AttributeError as e:
msg_dict = dict(url=request.url, e=e)
msg = _("%(url)s returned a fault: %(e)s") % msg_dict
LOG.info(msg)
if hasattr(response, 'headers'):
for hdr, val in response.headers.items():
# Headers must be utf-8 strings
response.headers[hdr] = six.text_type(val)
if not request.api_version_request.is_null():
response.headers[API_VERSION_REQUEST_HEADER] = (
request.api_version_request.get_string())
if request.api_version_request.experimental:
response.headers[EXPERIMENTAL_API_REQUEST_HEADER] = (
request.api_version_request.experimental)
response.headers['Vary'] = API_VERSION_REQUEST_HEADER
return response
def __init__(self,
name,
app,
host=None,
port=None,
pool_size=None,
protocol=eventlet.wsgi.HttpProtocol,
backlog=128):
"""Initialize, but do not start, a WSGI server.
:param name: Pretty name for logging.
:param app: The WSGI application to serve.
:param host: IP address to serve the application.
:param port: Port number to server the application.
:param pool_size: Maximum number of eventlets to spawn concurrently.
:returns: None
"""
eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
self.client_socket_timeout = CONF.client_socket_timeout
self.name = name
self.app = app
self._host = host or "0.0.0.0"
self._port = port or 0
self._server = None
self._socket = None
self._protocol = protocol
self.pool_size = pool_size or self.default_pool_size
self._pool = eventlet.GreenPool(self.pool_size)
self._logger = log.getLogger("eventlet.wsgi.server")
if backlog < 1:
raise exception.InvalidInput(
reason='The backlog must be more than 1')
bind_addr = (host, port)
# TODO(dims): eventlet's green dns/socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
try:
info = socket.getaddrinfo(bind_addr[0], bind_addr[1],
socket.AF_UNSPEC, socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
except Exception:
family = socket.AF_INET
cert_file = CONF.ssl_cert_file
key_file = CONF.ssl_key_file
ca_file = CONF.ssl_ca_file
self._use_ssl = cert_file or key_file
if cert_file and not os.path.exists(cert_file):
raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)
if ca_file and not os.path.exists(ca_file):
raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)
if key_file and not os.path.exists(key_file):
raise RuntimeError(_("Unable to find key_file : %s") % key_file)
if self._use_ssl and (not cert_file or not key_file):
raise RuntimeError(
_("When running server in SSL mode, you must "
"specify both a cert_file and key_file "
"option value in your configuration file"))
retry_until = time.time() + 30
while not self._socket and time.time() < retry_until:
try:
self._socket = eventlet.listen(bind_addr,
backlog=backlog,
family=family)
except socket.error as err:
if err.args[0] != errno.EADDRINUSE:
raise
eventlet.sleep(0.1)
if not self._socket:
raise RuntimeError(
_("Could not bind to %(host)s:%(port)s "
"after trying for 30 seconds") % {
'host': host,
'port': port
})
(self._host, self._port) = self._socket.getsockname()[0:2]
LOG.info("%(name)s listening on %(_host)s:%(_port)s", {
'name': self.name,
'_host': self._host,
'_port': self._port
})
