def limited(items, request, max_limit=CONF.osapi_max_limit): """Return a slice of items according to requested offset and limit. :param items: A sliceable entity :param request: ``wsgi.Request`` possibly containing 'offset' and 'limit' GET variables. 'offset' is where to start in the list, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default to max_limit. Negative values for either offset or limit will cause exc.HTTPBadRequest() exceptions to be raised. :kwarg max_limit: The maximum number of items to return from 'items' """ try: offset = int(request.GET.get('offset', 0)) except ValueError: msg = _('offset param must be an integer') raise webob.exc.HTTPBadRequest(explanation=msg) try: limit = int(request.GET.get('limit', max_limit)) except ValueError: msg = _('limit param must be an integer') raise webob.exc.HTTPBadRequest(explanation=msg) if limit < 0: msg = _('limit param must be positive') raise webob.exc.HTTPBadRequest(explanation=msg) if offset < 0: msg = _('offset param must be positive') raise webob.exc.HTTPBadRequest(explanation=msg) limit = min(max_limit, limit or max_limit) range_end = offset + limit return items[offset:range_end]
def wait_for_access_update(context, db, learning_instance, migration_wait_access_rules_timeout): starttime = time.time() deadline = starttime + migration_wait_access_rules_timeout tries = 0 while True: instance = db.learning_instance_get(context, learning_instance['id']) if instance['access_rules_status'] == constants.STATUS_ACTIVE: break tries += 1 now = time.time() if instance['access_rules_status'] == constants.STATUS_ERROR: msg = _("Failed to update access rules" " on learning instance %s") % learning_instance['id'] raise exception.LearningMigrationFailed(reason=msg) elif now > deadline: msg = _("Timeout trying to update access rules" " on learning instance %(learning_id)s. Timeout " "was %(timeout)s seconds.") % { 'learning_id': learning_instance['id'], 'timeout': migration_wait_access_rules_timeout } raise exception.LearningMigrationFailed(reason=msg) else: time.sleep(tries**2)
def main(): """Parse options and call the appropriate class/method.""" CONF.register_cli_opt(category_opt) script_name = sys.argv[0] if len(sys.argv) < 2: print( _("\nOpenStack meteos version: %(version)s\n") % {'version': version.version_string()}) print(script_name + " category action [<args>]") print(_("Available categories:")) for category in CATEGORIES: print("\t%s" % category) sys.exit(2) try: log.register_options(CONF) CONF(sys.argv[1:], project='meteos', version=version.version_string()) log.setup(CONF, "meteos") except cfg.ConfigFilesNotFoundError: cfgfile = CONF.config_file[-1] if CONF.config_file else None if cfgfile and not os.access(cfgfile, os.R_OK): st = os.stat(cfgfile) print(_("Could not read %s. Re-running with sudo") % cfgfile) try: os.execvp('sudo', ['sudo', '-u', '#%s' % st.st_uid] + sys.argv) except Exception: print(_('sudo failed, continuing as if nothing happened')) print(_('Please re-run meteos-manage as root.')) sys.exit(2) fn = CONF.category.action_fn fn_args = fetch_func_args(fn) fn(*fn_args)
def _get_limit_param(request): """Extract integer limit from request or fail.""" try: limit = int(request.GET['limit']) except ValueError: msg = _('limit param must be an integer') raise webob.exc.HTTPBadRequest(explanation=msg) if limit < 0: msg = _('limit param must be positive') raise webob.exc.HTTPBadRequest(explanation=msg) return limit
def validate_update(self, body, status_attr='status'): update = {} try: update[status_attr] = body[status_attr] except (TypeError, KeyError): msg = _("Must specify '%s'") % status_attr raise webob.exc.HTTPBadRequest(explanation=msg) if update[status_attr] not in self.valid_statuses[status_attr]: expl = (_("Invalid state. Valid states: %s.") % ", ".join( six.text_type(i) for i in self.valid_statuses[status_attr])) raise webob.exc.HTTPBadRequest(explanation=expl) return update
def is_admin_context(context): """Indicates if the request context is an administrator.""" if not context: warnings.warn(_('Use of empty request context is deprecated'), DeprecationWarning) raise Exception('die') return context.is_admin
def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: # Option 1: simple string res = 'message\n' # Option 2: a nicely formatted HTTP exception page res = exc.HTTPForbidden(detail='Nice try') # Option 3: a webob Response object (in case you need to play with # headers, or you want to be treated like an iterable, or or or) res = Response(); res.app_iter = open('somefile') # Option 4: any wsgi app to be run next res = self.application # Option 5: you can get a Response object for a wsgi app, too, to # play with headers etc res = req.get_response(self.application) # You can then just return your response... return res # ... or set req.response and return None. req.response = res See the end of http://pythonpaste.org/webob/modules/dec.html for more info. """ raise NotImplementedError(_('You must implement __call__'))
def load_model(self, context, id, dataset_format, model_type, job_template_id, experiment_id, cluster_id): """Load a Model""" policy.check_policy(context, 'model', 'load') model = { 'id': id, 'dataset_format': dataset_format, 'model_type': model_type, 'job_template_id': job_template_id, 'experiment_id': experiment_id, 'cluster_id': cluster_id } if not self._enable_load_model(context): msg = _("Can not load multiple models at the same time.") raise exception.InvalidLearning(reason=msg) try: self.engine_rpcapi.load_model(context, model) updates = {'status': constants.STATUS_ACTIVATING} LOG.info("Accepted load of model %s.", id) self.db.model_update(context, id, updates) except Exception: with excutils.save_and_reraise_exception(): updates = {'status': constants.STATUS_ERROR} self.db.model_update(context, id, updates)
def is_valid_status(resource_name, status, valid_statuses): if status not in valid_statuses: msg = _("%(resource_name)s status must be %(valid_statuses)s") % { "resource_name": resource_name, "valid_statuses": valid_statuses } raise exception.InvalidStatus(reason=msg)
def action_peek_json(body): """Determine action to invoke.""" try: decoded = jsonutils.loads(body) except ValueError: msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg) # Make sure there's exactly one key... if len(decoded) != 1: msg = _("too many body keys") raise exception.MalformedRequestBody(reason=msg) # Return the action and the decoded body... return list(decoded.keys())[0]
def create(self): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) look_for_keys = True if self.path_to_private_key: self.path_to_private_key = os.path.expanduser( self.path_to_private_key) look_for_keys = False elif self.password: look_for_keys = False try: ssh.connect(self.ip, port=self.port, username=self.login, password=self.password, key_filename=self.path_to_private_key, look_for_keys=look_for_keys, timeout=self.conn_timeout) # Paramiko by default sets the socket timeout to 0.1 seconds, # ignoring what we set through the sshclient. This doesn't help for # keeping long lived connections. Hence, we have to bypass it, by # overriding it after the transport is initialized. We are setting # the sockettimeout to None and setting a keepalive packet so that, # the server will keep the connection open. All that does is sent # a keepalive packet every ssh_conn_timeout seconds. if self.conn_timeout: transport = ssh.get_transport() transport.sock.settimeout(None) transport.set_keepalive(self.conn_timeout) return ssh except Exception as e: msg = _("Check whether private key or password are correctly " "set. Error connecting via ssh: %s") % e LOG.error(msg) raise exception.SSHException(msg)
def _wait_for_model_to_load(self, ip, port, unload=False): stdout = "" stderr = "" starttime = time.time() deadline = starttime + self.configuration.load_model_timeout interval = self.configuration.api_retry_interval tries = 0 while True: try: stdout, stderr = self._run_ssh( ip, ['netstat', '-tnl', '|', 'grep', port]) except processutils.ProcessExecutionError: pass if not stdout and unload: break if stdout and not unload: break tries += 1 now = time.time() if now > deadline: msg = _("Timeout trying to load/unload model " "%s") % id raise exception.Invalid(reason=msg) LOG.debug("Waiting for load/unload to complete") time.sleep(interval)
def get_job_result(self, context, job_id, template_id, cluster_id): stdout = "" stderr = "" starttime = time.time() deadline = starttime + self.configuration.execute_job_timeout interval = self.configuration.api_retry_interval tries = 0 while True: job = self.cluster_api.get_job(context, job_id) if job.info['status'] == const.STATUS_JOB_SUCCESS: stdout = self._get_job_result(context, template_id, cluster_id, job_id) break elif job.info['status'] == const.STATUS_JOB_ERROR: stderr = self._get_job_result(context, template_id, cluster_id, job_id) break tries += 1 now = time.time() if now > deadline: msg = _("Timeout trying to create experiment " "%s") % job_id raise exception.Invalid(reason=msg) LOG.debug("Waiting for job to complete: Current status: %s", job.info['status']) time.sleep(interval) return stdout, stderr
def list(self, zone=None): """Show a list of all physical hosts. Filter by zone. args: [zone] """ print("%-25s\t%-15s" % (_('host'), _('zone'))) ctxt = context.get_admin_context() services = db.service_get_all(ctxt) if zone: services = [ s for s in services if s['availability_zone']['name'] == zone ] hosts = [] for srv in services: if not [h for h in hosts if h['host'] == srv['host']]: hosts.append(srv) for h in hosts: print("%-25s\t%-15s" % (h['host'], h['availability_zone']['name']))
def matches_versioned_method(self, method): """Compares this version to that of a versioned method.""" if type(method) != versioned_method.VersionedMethod: msg = _('An API version request must be compared ' 'to a VersionedMethod object.') raise exception.InvalidParameterValue(err=msg) return self.matches(method.start_version, method.end_version, method.experimental)
def _model_evaluation_get_all_with_filters(context, project_id=None, filters=None, sort_key=None, sort_dir=None): if not sort_key: sort_key = 'created_at' if not sort_dir: sort_dir = 'desc' query = (_model_evaluation_get_query(context).join()) query = query.filter(models.Model_Evaluation.project_id == project_id) # Apply filters if not filters: filters = {} # Apply sorting if sort_dir.lower() not in ('desc', 'asc'): msg = _("Wrong sorting data provided: sort key is '%(sort_key)s' " "and sort direction is '%(sort_dir)s'.") % { "sort_key": sort_key, "sort_dir": sort_dir } raise exception.InvalidInput(reason=msg) def apply_sorting(model_evaluation, query): sort_attr = getattr(model_evaluation, sort_key) sort_method = getattr(sort_attr, sort_dir.lower()) return query.order_by(sort_method()) try: query = apply_sorting(models.Model_Evaluation, query) except AttributeError: msg = _("Wrong sorting key provided - '%s'.") % sort_key raise exception.InvalidInput(reason=msg) # Returns list of model_evaluations that satisfy filters. query = query.all() return query
class MeteosException(Exception): """Base Meteos Exception To correctly use this class, inherit from it and define a 'message' property. That message will get printf'd with the keyword arguments provided to the constructor. """ message = _("An unknown exception occurred.") code = 500 headers = {} safe = False def __init__(self, message=None, detail_data={}, **kwargs): self.kwargs = kwargs self.detail_data = detail_data if 'code' not in self.kwargs: try: self.kwargs['code'] = self.code except AttributeError: pass for k, v in self.kwargs.items(): if isinstance(v, Exception): self.kwargs[k] = six.text_type(v) if not message: try: message = self.message % kwargs except Exception: # kwargs doesn't match a variable in the message # log the issue and the kwargs LOG.exception('Exception in string format operation.') for name, value in kwargs.items(): LOG.error("%(name)s: %(value)s", { 'name': name, 'value': value }) if CONF.fatal_exception_format_errors: raise else: # at least get the core message out if something happened message = self.message elif isinstance(message, Exception): message = six.text_type(message) if re.match('.*[^\.]\.\.$', message): message = message[:-1] self.msg = message super(MeteosException, self).__init__(message)
def _load_auth_plugin(self): if self.admin_auth: return self.admin_auth self.auth_plugin = ks_loading.load_auth_from_conf_options( CONF, self.group) if self.url.find('v2') > -1: self.auth_plugin = v2.Token().load_from_options( **self.deprecated_opts_for_v2) else: self.auth_plugin = v3.Token().load_from_options(**self.opts_for_v3) if self.auth_plugin: return self.auth_plugin msg = _('Cannot load auth plugin for %s') % self.group raise self.exception_module.Unauthorized(message=msg)
def __get_backend(self): if not self.__backend: backend_name = CONF[self.__pivot] if backend_name not in self.__backends: raise exception.Error(_('Invalid backend: %s') % backend_name) backend = self.__backends[backend_name] if isinstance(backend, tuple): name = backend[0] fromlist = backend[1] else: name = backend fromlist = backend self.__backend = __import__(name, None, None, fromlist) LOG.debug('backend %s', self.__backend) return self.__backend
def __call__(self, req): user_id = req.headers.get('X_USER') user_id = req.headers.get('X_USER_ID', user_id) if user_id is None: LOG.debug("Neither X_USER_ID nor X_USER found in request") return webob.exc.HTTPUnauthorized() # get the roles roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')] if 'X_TENANT_ID' in req.headers: # This is the new header since Keystone went to ID/Name project_id = req.headers['X_TENANT_ID'] else: # This is for legacy compatibility project_id = req.headers['X_TENANT'] # Get the auth token auth_token = req.headers.get('X_AUTH_TOKEN', req.headers.get('X_STORAGE_TOKEN')) # Build a context, including the auth_token... remote_address = req.remote_addr if CONF.use_forwarded_for: remote_address = req.headers.get('X-Forwarded-For', remote_address) service_catalog = None if req.headers.get('X_SERVICE_CATALOG') is not None: try: catalog_header = req.headers.get('X_SERVICE_CATALOG') service_catalog = jsonutils.loads(catalog_header) except ValueError: raise webob.exc.HTTPInternalServerError( _('Invalid service catalog json.')) ctx = context.RequestContext(user_id, project_id, roles=roles, auth_token=auth_token, remote_address=remote_address, service_catalog=service_catalog) req.environ['meteos.context'] = ctx return self.application
def wait_for_cluster_create(self, context, id): starttime = time.time() deadline = starttime + self.configuration.create_experiment_timeout interval = self.configuration.api_retry_interval tries = 0 while True: cluster = self.cluster_api.get_cluster(context, id) if cluster.status == const.STATUS_SAHARA_ACTIVE: break tries += 1 now = time.time() if now > deadline: msg = _("Timeout trying to create experiment " "%s") % id raise exception.Invalid(reason=msg) LOG.debug("Waiting for cluster to complete: Current status: %s", cluster.status) time.sleep(interval)
def limited_by_marker(items, request, max_limit=CONF.osapi_max_limit): """Return a slice of items according to the requested marker and limit.""" params = get_pagination_params(request) limit = params.get('limit', max_limit) marker = params.get('marker') limit = min(max_limit, limit) start_index = 0 if marker: start_index = -1 for i, item in enumerate(items): if 'flavorid' in item: if item['flavorid'] == marker: start_index = i + 1 break elif item['id'] == marker or item.get('uuid') == marker: start_index = i + 1 break if start_index < 0: msg = _('marker [%s] not found') % marker raise webob.exc.HTTPBadRequest(explanation=msg) range_end = start_index + limit return items[start_index:range_end]
def list(self): """Show a list of all meteos services.""" ctxt = context.get_admin_context() services = db.service_get_all(ctxt) print_format = "%-16s %-36s %-16s %-10s %-5s %-10s" print(print_format % (_('Binary'), _('Host'), _('Zone'), _('Status'), _('State'), _('Updated At'))) for svc in services: alive = utils.service_is_up(svc) art = ":-)" if alive else "XXX" status = 'enabled' if svc['disabled']: status = 'disabled' print(print_format % ( svc['binary'], svc['host'].partition('.')[0], svc['availability_zone']['name'], status, art, svc['updated_at'], ))
class Duplicated(MeteosException): message = _("Duplicate entry") code = 409 safe = True
def experimental(self, value): if type(value) != bool: msg = _('The experimental property must be a bool value.') raise exception.InvalidParameterValue(err=msg) self._experimental = value
def _set_read_deleted(self, read_deleted): if read_deleted not in ('no', 'yes', 'only'): raise ValueError( _("read_deleted can only be one of 'no', " "'yes' or 'only', not %r") % read_deleted) self._read_deleted = read_deleted
def serve(server, workers=None): global _launcher if _launcher: raise RuntimeError(_('serve() can only be called once')) _launcher = service.launch(CONF, server, workers=workers)
def _from_json(self, datastring): try: return jsonutils.loads(datastring) except ValueError: msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg)
def _process_stack(self, request, action, action_args, content_type, body, accept): """Implement the processing stack.""" # Get the implementing method try: meth, extensions = self.get_method(request, action, content_type, body) except (AttributeError, TypeError): return Fault(webob.exc.HTTPNotFound()) except KeyError as ex: msg = _("There is no such action: %s") % ex.args[0] return Fault(webob.exc.HTTPBadRequest(explanation=msg)) except exception.MalformedRequestBody: msg = _("Malformed request body") return Fault(webob.exc.HTTPBadRequest(explanation=msg)) if body: msg = ("Action: '%(action)s', calling method: %(meth)s, body: " "%(body)s") % { 'action': action, 'body': six.text_type(body), 'meth': six.text_type(meth) } LOG.debug(strutils.mask_password(msg)) else: LOG.debug("Calling method '%(meth)s'", {'meth': six.text_type(meth)}) # Now, deserialize the request body... try: if content_type: contents = self.deserialize(meth, content_type, body) else: contents = {} except exception.InvalidContentType: msg = _("Unsupported Content-Type") return Fault(webob.exc.HTTPBadRequest(explanation=msg)) except exception.MalformedRequestBody: msg = _("Malformed request body") return Fault(webob.exc.HTTPBadRequest(explanation=msg)) # Update the action args action_args.update(contents) project_id = action_args.pop("project_id", None) context = request.environ.get('meteos.context') if (context and project_id and (project_id != context.project_id)): msg = _("Malformed request url") return Fault(webob.exc.HTTPBadRequest(explanation=msg)) # Run pre-processing extensions response, post = self.pre_process_extensions(extensions, request, action_args) if not response: try: with ResourceExceptionHandler(): action_result = self.dispatch(meth, request, action_args) except Fault as ex: response = ex if not response: # No exceptions; convert action_result into a # ResponseObject resp_obj = None if type(action_result) is dict or action_result is None: resp_obj = ResponseObject(action_result) elif isinstance(action_result, ResponseObject): resp_obj = action_result else: response = action_result # Run post-processing extensions if resp_obj: _set_request_id_header(request, resp_obj) # Do a preserialize to set up the response object serializers = getattr(meth, 'wsgi_serializers', {}) resp_obj._bind_method_serializers(serializers) if hasattr(meth, 'wsgi_code'): resp_obj._default_code = meth.wsgi_code resp_obj.preserialize(accept, self.default_serializers) # Process post-processing extensions response = self.post_process_extensions( post, resp_obj, request, action_args) if resp_obj and not response: response = resp_obj.serialize(request, accept, self.default_serializers) try: msg_dict = dict(url=request.url, status=response.status_int) msg = _("%(url)s returned with HTTP %(status)d") % msg_dict except AttributeError as e: msg_dict = dict(url=request.url, e=e) msg = _("%(url)s returned a fault: %(e)s") % msg_dict LOG.info(msg) if hasattr(response, 'headers'): for hdr, val in response.headers.items(): # Headers must be utf-8 strings response.headers[hdr] = six.text_type(val) if not request.api_version_request.is_null(): response.headers[API_VERSION_REQUEST_HEADER] = ( request.api_version_request.get_string()) if request.api_version_request.experimental: response.headers[EXPERIMENTAL_API_REQUEST_HEADER] = ( request.api_version_request.experimental) response.headers['Vary'] = API_VERSION_REQUEST_HEADER return response
def __init__(self, name, app, host=None, port=None, pool_size=None, protocol=eventlet.wsgi.HttpProtocol, backlog=128): """Initialize, but do not start, a WSGI server. :param name: Pretty name for logging. :param app: The WSGI application to serve. :param host: IP address to serve the application. :param port: Port number to server the application. :param pool_size: Maximum number of eventlets to spawn concurrently. :returns: None """ eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line self.client_socket_timeout = CONF.client_socket_timeout self.name = name self.app = app self._host = host or "0.0.0.0" self._port = port or 0 self._server = None self._socket = None self._protocol = protocol self.pool_size = pool_size or self.default_pool_size self._pool = eventlet.GreenPool(self.pool_size) self._logger = log.getLogger("eventlet.wsgi.server") if backlog < 1: raise exception.InvalidInput( reason='The backlog must be more than 1') bind_addr = (host, port) # TODO(dims): eventlet's green dns/socket module does not actually # support IPv6 in getaddrinfo(). We need to get around this in the # future or monitor upstream for a fix try: info = socket.getaddrinfo(bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)[0] family = info[0] bind_addr = info[-1] except Exception: family = socket.AF_INET cert_file = CONF.ssl_cert_file key_file = CONF.ssl_key_file ca_file = CONF.ssl_ca_file self._use_ssl = cert_file or key_file if cert_file and not os.path.exists(cert_file): raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) if ca_file and not os.path.exists(ca_file): raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) if key_file and not os.path.exists(key_file): raise RuntimeError(_("Unable to find key_file : %s") % key_file) if self._use_ssl and (not cert_file or not key_file): raise RuntimeError( _("When running server in SSL mode, you must " "specify both a cert_file and key_file " "option value in your configuration file")) retry_until = time.time() + 30 while not self._socket and time.time() < retry_until: try: self._socket = eventlet.listen(bind_addr, backlog=backlog, family=family) except socket.error as err: if err.args[0] != errno.EADDRINUSE: raise eventlet.sleep(0.1) if not self._socket: raise RuntimeError( _("Could not bind to %(host)s:%(port)s " "after trying for 30 seconds") % { 'host': host, 'port': port }) (self._host, self._port) = self._socket.getsockname()[0:2] LOG.info("%(name)s listening on %(_host)s:%(_port)s", { 'name': self.name, '_host': self._host, '_port': self._port })