def _get_absolute_url(self, url: str) -> str:
"""
This method returns absolute url from local or http urls.
"""
current_uri = uri_reference(self._url) if self._url else uri_reference(
str(self._httpx_response.url))
given_uri = uri_reference(url)
if given_uri.is_absolute():
_url = url
else:
uri = given_uri.resolve_with(current_uri)
uri = uri.copy_with(fragment=None)
_url = uri.unsplit()
logger.debug('returning computed absolute url: %s', _url)
return _url
def is_valid_uri(uri,
require_scheme=True,
allowed_schemes={'http', 'https'},
require_authority=True):
uri = uri_reference(uri).normalize()
validator = validators.Validator().allow_schemes(*allowed_schemes)
if require_scheme:
validator.require_presence_of('scheme')
if require_authority:
validator.require_presence_of('host')
validator.check_validity_of(
'scheme',
'host',
'port',
'path',
'query',
)
try:
validator.validate(uri)
except exceptions.ValidationError:
return False
return True
def parse_uri(value):
uri = rfc3986.uri_reference(value)
validator = rfc3986.validators.Validator().require_presence_of(
'scheme', 'host'
)
validator.validate(uri)
return value
def check_source_schema(source):
if uri_reference(source).scheme != "postgresql":
print(
'Error: Only "postgresql" is supported as the source database '
'currently',
file=sys.stderr)
sys.exit(1)
def test_http_link_active(content, link=None):
"link URL must be active"
from requests import get
from requests.exceptions import RequestException
from rfc3986 import is_valid_uri, uri_reference
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
if not is_valid_uri(value, require_scheme=True):
return
parsed_value = uri_reference(value)
if parsed_value.scheme not in ("http", "https"):
return
# Hooray.
if parsed_value.host.endswith("linkedin.com"):
raise SkipTest("linkedin.com won't let us see {} anyway".format(value))
try:
r = get(value, timeout=5.0, headers={"User-Agent": USER_AGENT})
except RequestException as exc:
assert False, "error while checking {}: {}".format(value, exc)
else:
assert 200 <= r.status_code < 300, \
"expected {} link {} to be active, but got {}".format(key, value, r.status_code)
def _handle_url(self, url: str) -> None:
if self._is_url_already_processed(url):
return
ur = uri_reference(url)
error_message = ''
if ur.scheme == 'file':
error_message = f'unable to open file {url}'
else:
if self._is_url_excluded_for_spider(url):
return
unreachable, fetch_time = self._get_resource(url, error_message)
if unreachable:
self.unreachable_urls.add(url)
return
# we update some stats
self.request_counter += 1
self._total_fetch_time += fetch_time
self.reachable_urls.add(url)
handle = self._driver.current_window_handle
try:
self.parse(self, self._get_selenium_response(handle))
except Exception:
logger.exception(
'something unexpected happened while parsing the content at url %s',
url)
if not self._ignore_errors:
raise
logger.info('content at url %s has been processed', url)
def __call__(self, value):
uri = rfc3986.uri_reference(value)
validator = rfc3986.validators.Validator().require_presence_of(
'scheme',
'host',
).check_validity_of(
'scheme',
'host',
'path',
)
if self.schemes:
validator = validator.allow_schemes(*self.schemes)
try:
validator.validate(uri)
except rfc3986.exceptions.RFC3986Exception as exc:
raise ValueError(exc)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
# NOTE(dhellmann): self.value is deprecated, and we don't want
# to trigger a deprecation warning ourselves so we modify
# self._value directly.
self._value = value
return value
def test_http_link_active(content, link=None):
"link URL must be active"
import cfscrape
from requests.exceptions import RequestException
from rfc3986 import is_valid_uri, uri_reference
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
if not is_valid_uri(value, require_scheme=True):
return
parsed_value = uri_reference(value)
if parsed_value.scheme not in ("http", "https"):
return
# Hooray.
if parsed_value.host.endswith("linkedin.com"):
raise SkipTest("linkedin.com won't let us see {} anyway".format(value))
try:
r = cfscrape.create_scraper().get(value, timeout=30.0, headers={"User-Agent": USER_AGENT})
except RequestException as exc:
assert False, "error while checking {}: {}".format(value, exc)
else:
assert 200 <= r.status_code < 300, \
"expected {} link {} to be active, but got {}".format(key, value, r.status_code)
def normalize(url: str, keep_utm: bool = True) -> str:
"""Normalize the provided URL.
:param url: The URL to be normalized.
:type url: str
:param keep_utm: Whether or not to keep campaign tracking query parameters.
:type keep_utm: bool
:returns: A normalized URL.
:rtype: str
"""
# We have to clean the host name first otherwise ``authority_info()`` bellow
# will complain.
scheme, netloc, *remaining = split(url)
url = rfc3986.uri_reference(
urllib.parse.urlunsplit((scheme, canonical_host(netloc), *remaining)))
# Make sure protocol's default port number is stripped away.
# TODO: Make sure to properly parse user credentials information.
auth = url.authority_info()
auth["port"] = canonical_port(auth["port"], url.scheme)
return url.copy_with(
scheme=normalizers.normalize_scheme(url.scheme or ""),
authority=normalizers.normalize_authority(
(auth["userinfo"], auth["host"], auth["port"])),
path=normalizers.normalize_path(canonical_path(url.path) or ""),
query=normalizers.normalize_query(
canonical_query(url.query, keep_utm=keep_utm)),
fragment=normalizers.normalize_fragment(url.fragment or None),
).unsplit()
def _get_absolute_url(self, url: str) -> str:
"""
This method returns absolute url from local or http urls.
"""
uri = uri_reference(url)
if uri.is_absolute():
_url = url
else:
# this is probably useless because I checked with firefox and chrome and both
# computes absolute urls when they encounter a relative one, but to remove any doubt, let's do this
current_uri = uri_reference(self.driver.current_url)
uri = uri.resolve_with(current_uri)
uri = uri.copy_with(fragment=None)
_url = uri.unsplit()
logger.debug('returning computed absolute url: %s', _url)
return _url
def validate_url(url: str):
"""
Validate the url against several checks and raises an exception in case of validation failure
:param url: a string with the url to be validated
:return: None
"""
validator.validate(uri_reference(url))
def is_valid_uri(uri):
try:
URI_VALIDATOR.validate(uri_reference(uri))
return True
except UriValidationError as exc:
LOGGER.debug(exc)
return False
def format_check_url(instance):
uri = uri_reference(instance)
try:
uri_validator.validate(uri)
except RFC3986Exception:
return False
return True
def validate_uri(self):
"""
Specification URIs have to be valid, have a scheme and be absolute
:rtype: bool
"""
uri = rfc3986.uri_reference(self.identifier)
valid_uri_scheme = uri.scheme in self.allowed_schemes
return uri.is_valid() and valid_uri_scheme and uri.is_absolute()
def test_invalid_uri_with_invalid_path(invalid_uri):
"""Verify we catch multiple invalid components."""
uri = rfc3986.uri_reference(invalid_uri)
uri = uri.copy_with(path='#foobar')
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of(
'host', 'path',
).validate(uri)
def validate_uri(self):
"""
Specification URIs have to be valid, have a scheme and be absolute
:rtype: bool
"""
uri = rfc3986.uri_reference(self.identifier)
valid_uri_scheme = uri.scheme in self.allowed_schemes
return uri.is_valid() and valid_uri_scheme and uri.is_absolute()
def test_invalid_uri_with_invalid_path(invalid_uri):
"""Verify we catch multiple invalid components."""
uri = rfc3986.uri_reference(invalid_uri)
uri = uri.copy_with(path="#foobar")
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of(
"host",
"path",
).validate(uri)
def create_stream(id, url):
if id and url and uri_reference(url).is_valid:
db = get_db()
c = db.cursor()
if len(c.execute('SELECT id,url FROM streams where id=:id', {'id':id}).fetchall()) == 0:
c.execute('INSERT into streams (id,url) VALUES (:id,:url)', {'id':id, 'url':url})
db.commit()
return True
return False
def format_check_uri_query_value(instance):
uri = uri_reference("http://example.com/path?{}#fragment".format(instance))
try:
uri_all_components_validator.validate(uri)
except RFC3986Exception:
return False
return (uri.scheme == "http" and uri.authority == "example.com"
and uri.path == "/path" and uri.query == instance
and uri.fragment == "fragment")
def test_link_value_https_preferred(content, link=None):
"link URL should use HTTPS whenever possible"
from rfc3986 import is_valid_uri, uri_reference
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
if is_valid_uri(value, require_scheme=True):
parsed_value = uri_reference(value)
if parsed_value.scheme == "http":
raise TestWarning("URL scheme is HTTP, but HTTPS is strongly preferred: {}".format(value))
def test_link_value_https_preferred(content, link=None):
"link URL should use HTTPS whenever possible"
from rfc3986 import is_valid_uri, uri_reference
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
if is_valid_uri(value, require_scheme=True):
parsed_value = uri_reference(value)
if parsed_value.scheme == "http":
raise TestWarning("URL scheme is HTTP, but HTTPS is strongly preferred: {}".format(value))
def format_check_uri_path(instance):
uri = uri_reference(
'http://example.com/{}?param=value#fragment'.format(instance))
try:
uri_all_components_validator.validate(uri)
except RFC3986Exception:
return False
return (uri.scheme == 'http' and uri.authority == 'example.com'
and uri.path == '/' + instance and uri.query == 'param=value'
and uri.fragment == 'fragment')
def test_from_uri_reference():
uri = uri_reference("http://foo.bar:1234/baz")
uribuilder = builder.URIBuilder().from_uri(uri)
assert uribuilder.scheme == "http"
assert uribuilder.userinfo is None
assert uribuilder.host == "foo.bar"
assert uribuilder.port == "1234"
assert uribuilder.path == "/baz"
assert uribuilder.query is None
assert uribuilder.fragment is None
def is_url(value):
ret = False
try:
if ((value and isinstance(value, six.string_types))
and rfc3986.is_valid_uri(value, require_scheme=True)
and rfc3986.uri_reference(value).scheme.lower()
in ['http', 'https']):
ret = True
except ValueError:
pass
return ret
def _validate_repository_url(repository_url: str) -> None:
"""Validate the given url for allowed schemes and components."""
# Allowed schemes are http and https, based on whether the repository
# supports TLS or not, and scheme and host must be present in the URL
validator = (rfc3986.validators.Validator().allow_schemes(
"http", "https").require_presence_of("scheme", "host"))
try:
validator.validate(rfc3986.uri_reference(repository_url))
except rfc3986.exceptions.RFC3986Exception as exc:
raise exceptions.UnreachableRepositoryURLDetected(
f"Invalid repository URL: {exc.args[0]}.")
def _validate_uri(instance):
uri = rfc3986.uri_reference(instance)
validator = rfc3986.validators.Validator().require_presence_of(
'scheme', 'host',
).check_validity_of(
'scheme', 'userinfo', 'host', 'path', 'query', 'fragment',
)
try:
validator.validate(uri)
except rfc3986.exceptions.RFC3986Exception:
return False
return True
def enforce_scheme(url: str, default_scheme: str) -> str:
"""
Makes sure the input url has a scheme. Otherise, add a default one.
:param url: the input url with or without a scheme.
:param default_scheme: default scheme to be added
:return: an url with a default scheme or the original one
"""
try:
scheme_presence_validator.validate(uri_reference(url))
return url
except ValidationError:
return "{}://{}".format(default_scheme, url)
def normalize_uri_result(uri):
"""
Normalize a URI (And return a URIResult)
"""
ref = uri_reference(uri).normalize()
return ref._replace(
authority=normalize_uri_authority(ref),
query=normalize_uri_query(ref),
path=normalize_uri_path(ref),
)
def notification(self, val: Union[uri_reference,
List[uri_reference]]) -> None:
if val is None:
self.__notification = []
elif isinstance(val, list):
r: List[str] = []
for x in val:
u = uri_reference(x)
if not u.is_valid():
raise WQXException(
"Attribute 'notification' must be a list of valid URIs, if provided."
)
r.append(u)
self.__notification = r
else:
u = uri_reference(val)
if not u.is_valid():
raise WQXException(
"Attribute 'notification' must be a list of valid URIs, if provided."
)
self.__notification = [u]
def test_validating_rfc_4007_ipv6_zone_ids():
"""Verify that RFC 4007 IPv6 Zone IDs are invalid
host/authority but after normalization are valid
"""
uri = rfc3986.uri_reference("http://[::1%eth0]")
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of("host").validate(uri)
uri = uri.normalize()
assert uri.host == "[::1%25eth0]"
validators.Validator().check_validity_of("host").validate(uri)
def is_valid_uri(uri, require_scheme={"http", "https"},
require_authority=True):
uri = uri_reference(uri).normalize()
if not uri.is_valid(require_scheme=require_scheme,
require_authority=require_authority):
return False
if require_scheme and not isinstance(require_scheme, bool):
if uri.scheme not in require_scheme:
return False
return True
def hosted_id_in_verification_scope(state, task_meta, **options):
try:
assertion_id = task_meta.get('node_id')
assertion_node = get_node_by_id(state, assertion_id)
badgeclass_node = get_node_by_id(state, assertion_node['badge'])
issuer_node = get_node_by_id(state, badgeclass_node['issuer'])
issuer_id = issuer_node.get('id')
except IndexError:
raise TaskPrerequisitesError()
try:
verification_policy = issuer_node.get('verification')
if verification_policy is None or isinstance(verification_policy,
six.string_types):
verification_policy = get_node_by_id(
state, issuer_node.get('verification'))
except IndexError:
verification_policy = _default_verification_policy(issuer_node)
if verification_policy.get('startsWith'):
starts_with = list_of(verification_policy['startsWith'])
if not any([assertion_id.startswith(i) for i in starts_with]):
return task_result(
False, "Assertion id {}".format(assertion_id) +
"does not start with any permitted values in its issuer's verification policy."
)
allowed_origins = list_of(
verification_policy.get(
'allowedOrigins',
_default_allowed_origins_for_issuer_id(issuer_id)))
if not allowed_origins or not issuer_id.startswith('http'):
return task_result(
True,
"There are no allowed hosted verification URL domains for this issuer's verification policy.",
actions=[
report_message(
'Issuer {} has no HTTP domain to enforce hosted verification policy against.'
.format(issuer_id),
message_level=MESSAGE_LEVEL_WARNING)
])
elif rfc3986.uri_reference(assertion_id).authority not in allowed_origins:
return task_result(
False, 'Assertion {} not hosted in allowed origins {}'.format(
abv(assertion_id), abv(allowed_origins)))
return task_result(
True,
'Assertion {} origin matches allowed value in issuer verification policy {}.'
.format(abv(assertion_id), abv(allowed_origins)))
def is_valid_uri(uri,
require_scheme={"http", "https"},
require_authority=True):
uri = uri_reference(uri).normalize()
if not uri.is_valid(require_scheme=require_scheme,
require_authority=require_authority):
return False
if require_scheme and not isinstance(require_scheme, bool):
if uri.scheme not in require_scheme:
return False
return True
def test_should_raise_error_when_item_in_the_list_has_an_invalid_component_part(
self, mocker):
# todo: like I said in the source code, I don't know how to reproduce this error, so for now
# I will just mock the validate method
url = 'http://fobor#ge.com'
uri = uri_reference('http://fobor#ge.com')
exception = InvalidComponentsError(uri, 'path')
mocker.patch('rfc3986.validators.Validator.validate',
side_effect=exception)
with pytest.raises(ValueError) as exc_info:
Spider([url], self.dummy_parse)
assert f'{url} is not a valid url' == str(exc_info.value)
def _handle_url(self, url: str) -> None:
if self._is_url_already_processed(url):
return
static_url = text = ''
response: Optional[httpx.Response] = None
ur = uri_reference(url)
if ur.scheme == 'file':
static_url = url
logger.debug(
'url %s is a file url so we attempt to read its content')
file_path = ur.path[1:] if platform.system(
) == 'Windows' else ur.path
try:
before = time()
with open_file(file_path) as f:
text = f.read()
fetch_time = time() - before
except OSError:
logger.exception('unable to open file %s', url)
self.unreachable_urls.add(url)
return
else:
if self._is_url_excluded_for_spider(url):
return
response: httpx.Response = self._fetch(url)
if response.is_error:
logger.info(
'fetching url %s returns an error with status code %s',
url, response.status_code)
self.unreachable_urls.add(url)
return
fetch_time = response.elapsed.total_seconds()
# we update some variables for statistics
self.request_counter += 1
self.reachable_urls.add(url)
self._total_fetch_time += fetch_time
try:
self.parse(self,
self._get_static_response(static_url, text, response))
except Exception:
logger.exception(
'something unexpected happened while parsing the content at url %s',
url)
if not self._ignore_errors:
raise
logger.info('content at url %s has been processed', url)
def check_target_schema(target):
uri = uri_reference(target)
if uri.scheme != "mysql+pymysql":
print(
'Error: Only "mysql" with the "pymysql" driver is supported '
'as the target database currently',
file=sys.stderr)
sys.exit(1)
if uri.query is None or "charset=utf8" not in uri.query:
print(
'Error: The target connection is missing the "charset=utf8" '
'parameter.',
file=sys.stderr)
sys.exit(1)
def test_validating_rfc_4007_ipv6_zone_ids():
"""Verify that RFC 4007 IPv6 Zone IDs are invalid
host/authority but after normalization are valid
"""
uri = rfc3986.uri_reference("http://[::1%eth0]")
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of(
'host'
).validate(uri)
uri = uri.normalize()
assert uri.host == '[::1%25eth0]'
validators.Validator().check_validity_of(
'host'
).validate(uri)
def __call__(self, value):
if not rfc3986.is_valid_uri(value, require_scheme=True,
require_authority=True):
raise ValueError('invalid URI: %r' % value)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
if self.schemes:
scheme = rfc3986.uri_reference(value).scheme
if scheme not in self.schemes:
raise ValueError("URI scheme '%s' not in %s" %
(scheme, self.schemes))
self.value = value
return value
def is_valid_uri(
uri, require_scheme=True, allowed_schemes={"http", "https"}, require_authority=True
):
uri = uri_reference(uri).normalize()
validator = validators.Validator().allow_schemes(*allowed_schemes)
if require_scheme:
validator.require_presence_of("scheme")
if require_authority:
validator.require_presence_of("host")
validator.check_validity_of("scheme", "host", "port", "path", "query")
try:
validator.validate(uri)
except exceptions.ValidationError:
return False
return True
def _validate_project_url(value):
try:
label, url = value.split(", ", 1)
except ValueError:
raise wtforms.validators.ValidationError("Must have both a label and an URL.") from None
if not label:
raise wtforms.validators.ValidationError("Must have a label.")
if len(label) > 32:
raise wtforms.validators.ValidationError("Label must not be longer than 32 characters.")
if not url:
raise wtforms.validators.ValidationError("Must have an URL.")
url = uri_reference(url)
url = url.normalize()
if not (url.is_valid() and url.scheme in ("http", "https")):
raise wtforms.validators.ValidationError("Invalid URL.")
def is_valid_uri(uri, require_scheme=True,
allowed_schemes={'http', 'https'},
require_authority=True):
uri = uri_reference(uri).normalize()
validator = validators.Validator().allow_schemes(*allowed_schemes)
if require_scheme:
validator.require_presence_of('scheme')
if require_authority:
validator.require_presence_of('host')
validator.check_validity_of(
'scheme', 'host', 'port', 'path', 'query',
)
try:
validator.validate(uri)
except exceptions.ValidationError:
return False
return True
def __call__(self, value):
uri = rfc3986.uri_reference(value)
validator = rfc3986.validators.Validator().require_presence_of(
'scheme', 'host',
).check_validity_of(
'scheme', 'host', 'path',
)
if self.schemes:
validator = validator.allow_schemes(*self.schemes)
try:
validator.validate(uri)
except rfc3986.exceptions.RFC3986Exception as exc:
raise ValueError(exc)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
# NOTE(dhellmann): self.value is deprecated, and we don't want
# to trigger a deprecation warning ourselves so we modify
# self._value directly.
self._value = value
return value
def test_invalid_uri_generates_error(invalid_uri):
"""Verify we catch invalid URIs."""
uri = rfc3986.uri_reference(invalid_uri)
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of('host').validate(uri)
def is_valid_openbazaar_scheme(uri):
"""Check for OpenBazaar appropriate scheme"""
return rfc3986.uri_reference(uri).scheme == u'tcp'
def test_unicode_uri_passed_as_bytes():
url_bytestring = b'http://example.com?utf8=\xe2\x98\x83'
uri = uri_reference(url_bytestring)
assert uri.is_valid() is True
assert uri == 'http://example.com?utf8=%E2%98%83'
def visit_dict(self, node):
if not os.path.basename(self.linter.current_file) in \
settings.MANIFEST_FILES \
or not isinstance(node.parent, astroid.Discard):
return
manifest_dict = ast.literal_eval(node.as_string())
# Check author is a string
author = manifest_dict.get('author', '')
if not isinstance(author, string_types):
self.add_message('manifest-author-string', node=node)
else:
# Check author required
authors = map(lambda author: author.strip(), author.split(','))
required_author = self.config.manifest_required_author
if required_author not in authors:
self.add_message('manifest-required-author', node=node,
args=(required_author,))
# Check keys required
required_keys = self.config.manifest_required_keys
for required_key in required_keys:
if required_key not in manifest_dict:
self.add_message('manifest-required-key', node=node,
args=(required_key,))
# Check keys deprecated
deprecated_keys = self.config.manifest_deprecated_keys
for deprecated_key in deprecated_keys:
if deprecated_key in manifest_dict:
self.add_message('manifest-deprecated-key', node=node,
args=(deprecated_key,))
# Check license allowed
license = manifest_dict.get('license', None)
if license and license not in self.config.license_allowed:
self.add_message('license-allowed',
node=node, args=(license,))
# Check version format
version_format = manifest_dict.get('version', '')
formatrgx = self.formatversion(version_format)
if version_format and not formatrgx:
self.add_message('manifest-version-format', node=node,
args=(version_format,
self.config.manifest_version_format_parsed))
# Check if resource exist
dirname = os.path.dirname(self.linter.current_file)
for key in DFTL_MANIFEST_DATA_KEYS:
for resource in (manifest_dict.get(key) or []):
if os.path.isfile(os.path.join(dirname, resource)):
continue
self.add_message('resource-not-exist', node=node,
args=(key, resource))
# Check if the website is valid URI
website = manifest_dict.get('website', '')
uri = rfc3986.uri_reference(website)
if ((website and ',' not in website) and
(not uri.is_valid(require_scheme=True,
require_authority=True) or
uri.scheme not in {"http", "https"})):
self.add_message('website-manifest-key-not-valid-uri',
node=node, args=(website))
def test_use_of_password():
"""Verify the behaviour of {forbid,allow}_use_of_password."""
validator = validators.Validator()
assert validator.allow_password is True
validator.forbid_use_of_password()
assert validator.allow_password is False
validator.allow_use_of_password()
assert validator.allow_password is True
@pytest.mark.parametrize('uri', [
rfc3986.uri_reference('https://*****:*****@github.com'),
rfc3986.uri_reference('https://*****:*****@github.com/path'),
rfc3986.uri_reference('https://*****:*****@github.com/path?query'),
rfc3986.uri_reference('https://*****:*****@github.com/path?query#frag'),
rfc3986.uri_reference('//user:[email protected]'),
])
def test_forbidden_passwords(uri):
"""Verify that passwords are disallowed."""
validator = validators.Validator().forbid_use_of_password()
with pytest.raises(exceptions.PasswordForbidden):
validator.validate(uri)
@pytest.mark.parametrize('uri', [
rfc3986.uri_reference('https://[email protected]'),
rfc3986.uri_reference('https://[email protected]/path'),
def test_unicode_authority():
url_bytestring = b'http://\xe2\x98\x83.com'
unicode_url = url_bytestring.decode('utf-8')
uri = uri_reference(unicode_url)
assert uri.is_valid() is False
assert uri == unicode_url
def test_unicode_uri():
url_bytestring = b'http://example.com?utf8=\xe2\x98\x83'
unicode_url = url_bytestring.decode('utf-8')
uri = uri_reference(unicode_url)
assert uri.is_valid() is True
assert uri == 'http://example.com?utf8=%E2%98%83'
def parse_driver_info(node):
"""Parse the information required for Ironic to connect to Redfish.
:param node: an Ironic node object
:returns: dictionary of parameters
:raises: InvalidParameterValue on malformed parameter(s)
:raises: MissingParameterValue on missing parameter(s)
"""
driver_info = node.driver_info or {}
missing_info = [key for key in REQUIRED_PROPERTIES
if not driver_info.get(key)]
if missing_info:
raise exception.MissingParameterValue(_(
'Missing the following Redfish properties in node '
'%(node)s driver_info: %(info)s') % {'node': node.uuid,
'info': missing_info})
# Validate the Redfish address
address = driver_info['redfish_address']
try:
parsed = rfc3986.uri_reference(address)
except TypeError:
raise exception.InvalidParameterValue(
_('Invalid Redfish address %(address)s set in '
'driver_info/redfish_address on node %(node)s') %
{'address': address, 'node': node.uuid})
if not parsed.scheme or not parsed.authority:
address = 'https://%s' % address
parsed = rfc3986.uri_reference(address)
if not parsed.is_valid(require_scheme=True, require_authority=True):
raise exception.InvalidParameterValue(
_('Invalid Redfish address %(address)s set in '
'driver_info/redfish_address on node %(node)s') %
{'address': address, 'node': node.uuid})
try:
system_id = urllib.parse.quote(driver_info['redfish_system_id'])
except (TypeError, AttributeError):
raise exception.InvalidParameterValue(
_('Invalid value "%(value)s" set in '
'driver_info/redfish_system_id on node %(node)s. '
'The value should be a path (string) to the resource '
'that the driver will interact with. For example: '
'/redfish/v1/Systems/1') %
{'value': driver_info['redfish_system_id'], 'node': node.uuid})
# Check if verify_ca is a Boolean or a file/directory in the file-system
verify_ca = driver_info.get('redfish_verify_ca', True)
if isinstance(verify_ca, six.string_types):
if os.path.isdir(verify_ca) or os.path.isfile(verify_ca):
pass
else:
try:
verify_ca = strutils.bool_from_string(verify_ca, strict=True)
except ValueError:
raise exception.InvalidParameterValue(
_('Invalid value type set in driver_info/'
'redfish_verify_ca on node %(node)s. '
'The value should be a Boolean or the path '
'to a file/directory, not "%(value)s"'
) % {'value': verify_ca, 'node': node.uuid})
elif isinstance(verify_ca, bool):
# If it's a boolean it's grand, we don't need to do anything
pass
else:
raise exception.InvalidParameterValue(
_('Invalid value type set in driver_info/redfish_verify_ca '
'on node %(node)s. The value should be a Boolean or the path '
'to a file/directory, not "%(value)s"') % {'value': verify_ca,
'node': node.uuid})
return {'address': address,
'system_id': system_id,
'username': driver_info.get('redfish_username'),
'password': driver_info.get('redfish_password'),
'verify_ca': verify_ca,
'node_uuid': node.uuid}
def test_unicode_uri():
url_bytestring = SNOWMAN_PARAMS
unicode_url = url_bytestring.decode('utf-8')
uri = uri_reference(unicode_url)
assert uri.is_valid() is True
assert uri == 'http://example.com?utf8=%E2%98%83'
def test_unicode_uri_passed_as_bytes():
url_bytestring = SNOWMAN_PARAMS
uri = uri_reference(url_bytestring)
assert uri.is_valid() is True
assert uri == 'http://example.com?utf8=%E2%98%83'
def test_unicode_authority():
url_bytestring = SNOWMAN_HOST
unicode_url = url_bytestring.decode('utf-8')
uri = uri_reference(unicode_url)
assert uri.is_valid() is False
assert uri == unicode_url
