def test_ost_version(self):
ost_version = '2019-05-01T19:53:21.498745'
self.assertEqual(
ost_version, os_service_types.ServiceTypes().version,
"This project must be pinned to the latest version of "
"os-service-types. Please bump requirements.txt and "
"lower-constraints.txt accordingly.")
def main():
projects = yaml.load(open('../../openstack/project-config/gerrit/projects.yaml'))
governance = yaml.load(open('../../openstack/governance/reference/projects.yaml'))
service_types = os_service_types.ServiceTypes()
tags = dict()
full_group_list = set()
manifest = ET.Element('manifest')
github_remote = ET.SubElement(manifest, 'remote')
github_remote.set('name', 'github')
github_remote.set('fetch', 'ssh://[email protected]/')
github_remote.set('review', 'https://review.openstack.org')
git_o_o_remote = ET.SubElement(manifest, 'remote')
git_o_o_remote.set('name', 'openstack')
git_o_o_remote.set('fetch', 'https://git.openstack.org/')
git_o_o_remote.set('review', 'https://review.openstack.org/')
default = ET.SubElement(manifest, 'default')
default.set('revision', 'master')
default.set('remote', 'openstack')
for project in governance:
types = service_types.get_all_service_data_for_project(project)
for deliverable in governance[project]['deliverables']:
for repo in governance[project]['deliverables'][deliverable]['repos']:
tags[repo] = [project.replace(' ', '-').lower(), 'offical']
if types:
for i in types:
tags[repo].append(i['service_type'])
if 'tags' in governance[project]['deliverables'][deliverable]:
for tag in governance[project]['deliverables'][deliverable]['tags']:
tags[repo].append(tag)
for project in projects:
groups = []
if project['project'].split('/')[0] not in ["openstack-attic", "stackforge"]:
project_fragment = ET.SubElement(manifest, 'project')
project_fragment.set("path", project['project'])
project_fragment.set("name", "%s.git" % project['project'])
groups = groups + tags.get(project['project'], [])
project_fragment.set("groups", "%s" % ','.join(groups))
[full_group_list.add(item) for item in groups]
try:
os.remove("groups.rst")
except OSError:
pass
with open("groups.rst", "a") as groups_file:
for group in sorted(full_group_list):
groups_file.write("* %s \n" % group)
with open("default.xml", "wb") as manifest_file:
indent(manifest)
tree = ET.ElementTree(manifest)
tree.write(manifest_file, xml_declaration=True,
encoding='utf-8', method="xml")
def main():
"""Entry point for this script."""
args = parse_command_line_arguments()
logger = initialize_logging(args.debug, args.verbose)
try:
loader = jinja2.FileSystemLoader(args.source_directory)
environment = jinja2.Environment(loader=loader)
except Exception as e:
logger.error("initialising template environment failed: %s" % e)
raise
try:
service_types = os_service_types.ServiceTypes(
session=requests.Session(), only_remote=True)
except Exception as e:
logger.error("initialising service types data failed: %s" % e)
return 1
for templateFile in environment.list_templates():
if not (templateFile.endswith('.html')
or templateFile.endswith('.htaccess')):
continue
logger.info("generating %s" % templateFile)
try:
template = environment.get_template(templateFile)
except Exception as e:
logger.error("parsing template %s failed: %s" % (templateFile, e))
raise
try:
if templateFile.endswith('.html'):
output = lxml.html.tostring(lxml.html.fromstring(
template.render()),
pretty_print=True)
else:
output = template.render(REVERSE=service_types.reverse)
except Exception as e:
logger.error("rendering template %s failed: %s" %
(templateFile, e))
raise
try:
target_directory = os.path.join(args.output_directory,
os.path.dirname(templateFile))
target_file = os.path.join(args.output_directory, templateFile)
if not os.path.isdir(target_directory):
logger.debug("creating target directory %s" % target_directory)
os.makedirs(target_directory)
logger.debug("writing %s" % target_file)
with open(os.path.join(target_file), 'wb') as fh:
fh.write(output.encode('utf8'))
except (IOError, OSError, UnicodeEncodeError) as e:
logger.error("writing %s failed: %s" % (target_file, e))
raise
return 0
def __init__(self,
name=None,
region_name=None,
config=None,
force_ipv4=False,
auth_plugin=None,
openstack_config=None,
session_constructor=None,
app_name=None,
app_version=None,
session=None,
discovery_cache=None,
extra_config=None,
cache_expiration_time=0,
cache_expirations=None,
cache_path=None,
cache_class='dogpile.cache.null',
cache_arguments=None,
password_callback=None,
statsd_host=None,
statsd_port=None,
statsd_prefix=None,
influxdb_config=None,
collector_registry=None):
self._name = name
self.config = _util.normalize_keys(config)
# NOTE(efried): For backward compatibility: a) continue to accept the
# region_name kwarg; b) make it take precedence over (non-service_type-
# specific) region_name set in the config dict.
if region_name is not None:
self.config['region_name'] = region_name
self._extra_config = extra_config or {}
self.log = _log.setup_logging('openstack.config')
self._force_ipv4 = force_ipv4
self._auth = auth_plugin
self._openstack_config = openstack_config
self._keystone_session = session
self._session_constructor = session_constructor or ks_session.Session
self._app_name = app_name
self._app_version = app_version
self._discovery_cache = discovery_cache or None
self._cache_expiration_time = cache_expiration_time
self._cache_expirations = cache_expirations or {}
self._cache_path = cache_path
self._cache_class = cache_class
self._cache_arguments = cache_arguments
self._password_callback = password_callback
self._statsd_host = statsd_host
self._statsd_port = statsd_port
self._statsd_prefix = statsd_prefix
self._statsd_client = None
self._influxdb_config = influxdb_config
self._influxdb_client = None
self._collector_registry = collector_registry
self._service_type_manager = os_service_types.ServiceTypes()
def take_action(self, parsed_args):
interface = parsed_args.interface
if parsed_args.is_all_interfaces:
interface = None
session = self.app.client_manager.session
version_data = session.get_all_version_data(
interface=interface,
region_name=parsed_args.region_name)
columns = [
"Region Name",
"Service Type",
"Version",
"Status",
"Endpoint",
"Min Microversion",
"Max Microversion",
]
status = parsed_args.status
if status:
status = status.upper()
service = parsed_args.service
if service:
# Normalize service type argument to official type
service_type_manager = os_service_types.ServiceTypes()
service = service_type_manager.get_service_type(service)
versions = []
for region_name, interfaces in version_data.items():
for interface, services in interfaces.items():
for service_type, service_versions in services.items():
if service and service != service_type:
# TODO(mordred) Once there is a version of
# keystoneauth that can do this filtering
# before making all the discovery calls, switch
# to that.
continue
for data in service_versions:
if status and status != data['status']:
continue
versions.append((
region_name,
service_type,
data['version'],
data['status'],
data['url'],
data['min_microversion'],
data['max_microversion'],
))
return (columns, versions)
def setUp(self):
super(TestRemote, self).setUp()
# Set up a requests_mock fixture for all HTTP traffic
adapter = self.useFixture(rm_fixture.Fixture())
adapter.register_uri('GET',
os_service_types.service_types.SERVICE_TYPES_URL,
json=self.remote_content,
headers={'etag': self.getUniqueString('etag')})
# Make an object that fetches from the network
self.service_types = os_service_types.ServiceTypes(
session=self.session)
self.assertEqual(1, len(adapter.request_history))
def __init__(self,
name=None,
region_name=None,
config=None,
force_ipv4=False,
auth_plugin=None,
openstack_config=None,
session_constructor=None,
app_name=None,
app_version=None,
session=None,
discovery_cache=None,
extra_config=None,
cache_expiration_time=0,
cache_expirations=None,
cache_path=None,
cache_class='dogpile.cache.null',
cache_arguments=None,
password_callback=None,
statsd_host=None,
statsd_port=None,
statsd_prefix=None,
collector_registry=None):
self._name = name
self.region_name = region_name
self.config = _util.normalize_keys(config)
self._extra_config = extra_config or {}
self.log = _log.setup_logging('openstack.config')
self._force_ipv4 = force_ipv4
self._auth = auth_plugin
self._openstack_config = openstack_config
self._keystone_session = session
self._session_constructor = session_constructor or ks_session.Session
self._app_name = app_name
self._app_version = app_version
self._discovery_cache = discovery_cache or None
self._cache_expiration_time = cache_expiration_time
self._cache_expirations = cache_expirations or {}
self._cache_path = cache_path
self._cache_class = cache_class
self._cache_arguments = cache_arguments
self._password_callback = password_callback
self._statsd_host = statsd_host
self._statsd_port = statsd_port
self._statsd_prefix = statsd_prefix
self._statsd_client = None
self._collector_registry = collector_registry
self._service_type_manager = os_service_types.ServiceTypes()
def _get_service_interfaces(self):
interfaces = {}
if not os_service_types:
return interfaces
types = os_service_types.ServiceTypes()
for name, _ in config.list_opts():
if not name or not name.startswith('clients_'):
continue
project_name = name.split("_", 1)[0]
service_data = types.get_service_data_for_project(project_name)
if not service_data:
continue
service_type = service_data['service_type']
interfaces[service_type + '_interface'] = self._get_client_option(
service_type, 'endpoint_type')
return interfaces
def validate_service_types():
print("Validating Service Types")
print("========================")
count = 0
# Load the current service-type-authority data
service_types = os_service_types.ServiceTypes(session=requests.Session(),
only_remote=True)
# Load the project job definitions
with io.open('jenkins/jobs/projects.yaml', 'r', encoding='utf-8') as f:
file_contents = local_yaml.load(f.read())
for item in file_contents:
project = item.get('project', {})
for job in project.get('jobs', []):
for api_job in _API_JOBS:
if api_job not in job:
continue
proj_data = service_types.get_service_data_for_project(
project['name'])
if not proj_data:
print('ERROR: Found service type reference "{}" for {} in '
'{} but not in authority list {}'.format(
job[api_job]['service'], api_job,
project['name'], service_types.url))
count += 1
else:
actual = job[api_job]['service']
expected = proj_data['service_type']
if actual != expected:
print('ERROR: Found service "{}" for {} '
'in {} but expected "{}"'.format(
job[api_job]['service'], api_job,
project['name'], expected))
count += 1
print('Found {} errors in service type settings '
'in jenkins/jobs/projects.yaml\n'.format(count))
return count
# License for the specific language governing permissions and limitations
# under the License.
import warnings
import os_service_types
from openstack import _log
from openstack import exceptions
__all__ = [
'ServiceDescription',
]
_logger = _log.setup_logging('openstack')
_service_type_manager = os_service_types.ServiceTypes()
class _ServiceDisabledProxyShim(object):
def __init__(self, service_type, reason):
self.service_type = service_type
self.reason = reason
def __getattr__(self, item):
raise exceptions.ServiceDisabledException(
"Service '{service_type}' is disabled because its configuration "
"could not be loaded. {reason}".format(
service_type=self.service_type, reason=self.reason or ''))
class ServiceDescription(object):
def load_project_data(source_directory,
check_all_links=False,
skip_links=False,
series_to_load=None,
governed_deliverables=[],
strict=False,
projects_to_check=[]):
"Return a dict with project data grouped by series."
logger = logging.getLogger()
series_to_load = series_to_load or []
project_data = {}
fail = False
service_types = os_service_types.ServiceTypes(session=requests.Session(),
only_remote=True)
# Set up a schema validator so we can quickly check that the input
# data conforms.
project_schema_filename = os.path.join(
source_directory,
'project-data',
'schema.yaml',
)
with open(project_schema_filename, 'r') as f:
project_schema = yaml.safe_load(f.read())
validator = jsonschema.Draft4Validator(project_schema)
# Load the data files, using the file basename as the release
# series name.
for filename in glob.glob(
os.path.join(source_directory, 'project-data', '*.yaml')):
if filename.endswith('schema.yaml'):
continue
series, _ = os.path.splitext(os.path.basename(filename))
if series_to_load and series not in series_to_load:
continue
logger.info('loading %s project data from %s', series, filename)
with open(filename, 'r') as f:
raw_data = yaml.safe_load(f.read())
for error in validator.iter_errors(raw_data):
logger.error(str(error))
fail = True
links_to_check = []
data = []
for project in raw_data:
deliverable_name = project.get('deliverable-name', project['name'])
# Set the defaults for the flags so that the templates can
# assume the flags with true defaults are defined.
for url_info in _URLS:
if url_info.flag_name not in project:
project[url_info.flag_name] = url_info.default
if (series == 'latest'
and deliverable_name not in governed_deliverables):
msg = ('{} is no longer part of an official project, '
'{} in {}').format(deliverable_name,
'error' if strict else 'ignoring',
filename)
logger.warning(msg)
if strict:
logger.info('Known deliverables: %s',
sorted(governed_deliverables))
raise RuntimeError(msg)
continue
logger.info('including %s', deliverable_name)
data.append(project)
# If the project has a service-type set, ensure it matches
# the value in the service-type-authority data.base.
st = project.get('service_type')
if st is not None:
st_data = service_types.get_service_data_for_project(
project['name'])
if not st_data:
# It's possible this is a project listed by its
# service-type
st_data = service_types.get_service_data(st)
if not st_data:
logger.error(
'did not find %s in Service Types Authority',
project['name'],
)
fail = True
elif st != st_data['service_type']:
logger.error(
'expected service_type %r for %s but got %r',
st_data['service_type'],
project['name'],
st,
)
fail = True
# client projects must have a description
project_type = project.get('type')
if (project_type in ['cloud-client', 'service-client']
and not project.get('description')):
logger.error(
'client project %s has no description',
project['name'],
)
fail = True
# If the project claims to have a separately published guide
# of some sort, look for it before allowing the flag to stand.
check_links_this_project = (deliverable_name in projects_to_check
or not projects_to_check)
if check_links_this_project and not skip_links:
for url_info in _URLS:
flag_val = project.get(url_info.flag_name,
url_info.default)
if ((not flag_val) and url_info.types
and project_type not in url_info.types):
# This type of project isn't expected to have
# this type of link, so if we are not
# explicitly told to check for it don't.
continue
try:
url = url_info.template.format(series=series,
**project)
except KeyError:
# The project data does not include a field needed
# to build the URL (typically the
# service_type). Ignore this URL, unless the flag
# is set.
if flag_val:
raise
continue
# Only try to fetch the URL if we're going to do
# something with the result.
if flag_val or check_all_links:
logger.info('%s:%s looking for %s', series,
project['name'], url)
links_to_check.append((url, project['name'],
url_info.flag_name, flag_val))
if links_to_check:
logger.info('checking %s links from %s...', len(links_to_check),
filename)
pool = multiprocessing.pool.ThreadPool()
results = pool.map(_check_url, links_to_check)
for url, project_name, flag, flag_val, exists, status in results:
if flag_val and not exists:
logger.error(
'%s set for %s but %s does not exist (%s)',
flag,
project_name,
url,
status,
)
fail = True
elif (not flag_val) and check_all_links and exists:
msg = '{} not set for {} but {} does exist'.format(
flag, project_name, url)
logger.warning(msg)
if strict:
raise RuntimeError(msg)
if fail:
raise ValueError('invalid input in %s' % filename)
project_data[series] = data
return project_data
def test_singleton_different(self):
service_types = os_service_types.ServiceTypes()
self.assertFalse(service_types is self.service_types)
def setUp(self):
super(TestWarnOff, self).setUp()
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
self.service_types = os_service_types.ServiceTypes()
def __init__(self, cloud=None, config=None, session=None,
app_name=None, app_version=None,
# TODO(shade) Remove these once we've shifted
# python-openstackclient to not use the profile interface.
authenticator=None, profile=None,
extra_services=None,
**kwargs):
"""Create a connection to a cloud.
A connection needs information about how to connect, how to
authenticate and how to select the appropriate services to use.
The recommended way to provide this information is by referencing
a named cloud config from an existing `clouds.yaml` file. The cloud
name ``envvars`` may be used to consume a cloud configured via ``OS_``
environment variables.
A pre-existing :class:`~openstack.config.cloud_region.CloudRegion`
object can be passed in lieu of a cloud name, for cases where the user
already has a fully formed CloudRegion and just wants to use it.
Similarly, if for some reason the user already has a
:class:`~keystoneauth1.session.Session` and wants to use it, it may be
passed in.
:param str cloud: Name of the cloud from config to use.
:param config: CloudRegion object representing the config for the
region of the cloud in question.
:type config: :class:`~openstack.config.cloud_region.CloudRegion`
:param session: A session object compatible with
:class:`~keystoneauth1.session.Session`.
:type session: :class:`~keystoneauth1.session.Session`
:param str app_name: Name of the application to be added to User Agent.
:param str app_version: Version of the application to be added to
User Agent.
:param authenticator: DEPRECATED. Only exists for short-term backwards
compatibility for python-openstackclient while we
transition. See `Transition from Profile`_ for
details.
:param profile: DEPRECATED. Only exists for short-term backwards
compatibility for python-openstackclient while we
transition. See `Transition from Profile`_ for details.
:param extra_services: List of
:class:`~openstack.service_description.ServiceDescription`
objects describing services that openstacksdk otherwise does not
know about.
:param kwargs: If a config is not provided, the rest of the parameters
provided are assumed to be arguments to be passed to the
CloudRegion contructor.
"""
self.config = config
self._extra_services = {}
if extra_services:
for service in extra_services:
self._extra_services[service.service_type] = service
if not self.config:
if profile:
import openstack.profile
# TODO(shade) Remove this once we've shifted
# python-openstackclient to not use the profile interface.
self.config = openstack.profile._get_config_from_profile(
profile, authenticator, **kwargs)
else:
openstack_config = _config.OpenStackConfig(
app_name=app_name, app_version=app_version,
load_yaml_config=profile is None)
self.config = openstack_config.get_one(
cloud=cloud, validate=session is None, **kwargs)
if self.config.name:
tm_name = ':'.join([
self.config.name,
self.config.region_name or 'unknown'])
else:
tm_name = self.config.region_name or 'unknown'
self.task_manager = task_manager.TaskManager(name=tm_name)
if session:
# TODO(mordred) Expose constructor option for this in OCC
self.config._keystone_session = session
self.session = self.config.get_session()
service_type_manager = os_service_types.ServiceTypes()
for service in service_type_manager.services:
self.add_service(
service_description.OpenStackServiceDescription(
service, self.config))
def from_conf(conf, session=None, service_types=None, **kwargs):
"""Create a CloudRegion from oslo.config ConfigOpts.
:param oslo_config.cfg.ConfigOpts conf:
An oslo.config ConfigOpts containing keystoneauth1.Adapter options in
sections named according to project (e.g. [nova], not [compute]).
TODO: Current behavior is to use defaults if no such section exists,
which may not be what we want long term.
:param keystoneauth1.session.Session session:
An existing authenticated Session to use. This is currently required.
TODO: Load this (and auth) from the conf.
:param service_types:
A list/set of service types for which to look for and process config
opts. If None, all known service types are processed. Note that we will
not error if a supplied service type can not be processed successfully
(unless you try to use the proxy, of course). This tolerates uses where
the consuming code has paths for a given service, but those paths are
not exercised for given end user setups, and we do not want to generate
errors for e.g. missing/invalid conf sections in those cases. We also
don't check to make sure your service types are spelled correctly -
caveat implementor.
:param kwargs:
Additional keyword arguments to be passed directly to the CloudRegion
constructor.
:raise openstack.exceptions.ConfigException:
If session is not specified.
:return:
An openstack.config.cloud_region.CloudRegion.
"""
if not session:
# TODO(mordred) Fill this in - not needed for first stab with nova
raise exceptions.ConfigException("A Session must be supplied.")
config_dict = kwargs.pop('config', config_defaults.get_defaults())
stm = os_service_types.ServiceTypes()
for st in stm.all_types_by_service_type:
if service_types is not None and st not in service_types:
_disable_service(
config_dict, st,
reason="Not in the list of requested service_types.")
continue
project_name = stm.get_project_name(st)
if project_name not in conf:
if '-' in project_name:
project_name = project_name.replace('-', '_')
if project_name not in conf:
_disable_service(
config_dict, st,
reason="No section for project '{project}' (service type "
"'{service_type}') was present in the config."
.format(project=project_name, service_type=st))
continue
opt_dict = {}
# Populate opt_dict with (appropriately processed) Adapter conf opts
try:
ks_load_adap.process_conf_options(conf[project_name], opt_dict)
except Exception as e:
# NOTE(efried): This is for (at least) a couple of scenarios:
# (1) oslo_config.cfg.NoSuchOptError when ksa adapter opts are not
# registered in this section.
# (2) TypeError, when opts are registered but bogus (e.g.
# 'interface' and 'valid_interfaces' are both present).
# We may want to consider (providing a kwarg giving the caller the
# option of) blowing up right away for (2) rather than letting them
# get all the way to the point of trying the service and having
# *that* blow up.
reason = ("Encountered an exception attempting to process config "
"for project '{project}' (service type "
"'{service_type}'): {exception}".format(
project=project_name, service_type=st, exception=e))
_logger.warning("Disabling service '{service_type}': "
"{reason}".format(service_type=st, reason=reason))
_disable_service(config_dict, st, reason=reason)
continue
# Load them into config_dict under keys prefixed by ${service_type}_
for raw_name, opt_val in opt_dict.items():
config_name = _make_key(raw_name, st)
config_dict[config_name] = opt_val
return CloudRegion(
session=session, config=config_dict, **kwargs)
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os_service_types
from keystoneauth1.exceptions import base
_SERVICE_TYPES = os_service_types.ServiceTypes()
__all__ = ('DiscoveryFailure', 'ImpliedVersionMismatch',
'ImpliedMinVersionMismatch', 'ImpliedMaxVersionMismatch',
'VersionNotAvailable')
class DiscoveryFailure(base.ClientException):
message = "Discovery of client versions failed."
class VersionNotAvailable(DiscoveryFailure):
message = "Discovery failed. Requested version is not available."
class ImpliedVersionMismatch(ValueError):
def setUp(self):
super(TestMatch, self).setUp()
# Make an object with no network access
self.service_types = os_service_types.ServiceTypes()