def test_missing_host_component(uri):
"""Verify that missing host components cause errors."""
validators.Validator().validate(uri)
validator = validators.Validator().require_presence_of("host")
with pytest.raises(exceptions.MissingComponentError):
validator.validate(uri)
def test_validating_rfc_4007_ipv6_zone_ids():
"""Verify that RFC 4007 IPv6 Zone IDs are invalid
host/authority but after normalization are valid
"""
uri = rfc3986.uri_reference("http://[::1%eth0]")
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of("host").validate(uri)
uri = uri.normalize()
assert uri.host == "[::1%25eth0]"
validators.Validator().check_validity_of("host").validate(uri)
def is_valid_uri(uri,
require_scheme=True,
allowed_schemes={'http', 'https'},
require_authority=True):
uri = uri_reference(uri).normalize()
validator = validators.Validator().allow_schemes(*allowed_schemes)
if require_scheme:
validator.require_presence_of('scheme')
if require_authority:
validator.require_presence_of('host')
validator.check_validity_of(
'scheme',
'host',
'port',
'path',
'query',
)
try:
validator.validate(uri)
except exceptions.ValidationError:
return False
return True
def test_multiple_missing_components(uri):
"""Verify that multiple missing components are caught."""
validator = validators.Validator().require_presence_of("scheme", "path")
with pytest.raises(exceptions.MissingComponentError) as captured_exc:
validator.validate(uri)
exception = captured_exc.value
assert 2 == len(exception.args[-1])
def test_successful_complex_validation(uri):
"""Verify we do not raise ValidationErrors for good URIs."""
validators.Validator().allow_schemes(
"https",
"ssh",
).allow_hosts(
"github.com",
"bitbucket.org",
"gitlab.com",
"git.openstack.org",
).allow_ports(
"22",
"443",
).require_presence_of(
"scheme",
"host",
"path",
).check_validity_of(
"scheme",
"userinfo",
"host",
"port",
"path",
"query",
"fragment",
).validate(uri)
def test_successful_complex_validation(uri):
"""Verify we do not raise ValidationErrors for good URIs."""
validators.Validator().allow_schemes(
'https',
'ssh',
).allow_hosts(
'github.com',
'bitbucket.org',
'gitlab.com',
'git.openstack.org',
).allow_ports(
'22',
'443',
).require_presence_of(
'scheme',
'host',
'path',
).check_validity_of(
'scheme',
'userinfo',
'host',
'port',
'path',
'query',
'fragment',
).validate(uri)
def validate_identifier(cls, identifier: Union[str, URIReference]) -> str:
"""
Validate an identifier and convert it into an URI string.
Args:
identifier: Either a full URI or the identifier part.
Raises:
`ValueError` if the identifier is invalid.
"""
uri = URIBuilder.from_uri(identifier)
if not uri.host:
if uri.path and not uri.path.startswith("/"):
uri = uri.add_path(f"/{uri.path}")
uri = uri.add_host(cls.DEFAULT_HOST)
if not uri.scheme:
uri = uri.add_scheme(cls.DEFAULT_SCHEME)
uri = uri.finalize()
validator = validators.Validator().require_presence_of("path")
validator = validator.allow_schemes(*cls.ALLOWED_SCHEMES)
validator = validator.allow_hosts(*cls.ALLOWED_HOSTS)
try:
validator.validate(uri)
except exceptions.RFC3986Exception as ex:
raise ValueError(
f"Identifier '{uri.unsplit()}' is invalid: {ex.args[0]}"
) from ex
else:
return uri.unsplit()
def test_invalid_uri_with_invalid_path(invalid_uri):
"""Verify we catch multiple invalid components."""
uri = rfc3986.uri_reference(invalid_uri)
uri = uri.copy_with(path="#foobar")
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of(
"host",
"path",
).validate(uri)
def test_allowing_hosts():
"""Verify the ability to select hosts to be allowed."""
validator = validators.Validator().allow_hosts(
"pypi.python.org",
"pypi.org",
)
assert "pypi.python.org" in validator.allowed_hosts
assert "pypi.org" in validator.allowed_hosts
def test_use_of_password():
"""Verify the behaviour of {forbid,allow}_use_of_password."""
validator = validators.Validator()
assert validator.allow_password is True
validator.forbid_use_of_password()
assert validator.allow_password is False
validator.allow_use_of_password()
assert validator.allow_password is True
def test_defaults():
"""Verify the default Validator settings."""
validator = validators.Validator()
assert validator.required_components == {
c: False
for c in validator.COMPONENT_NAMES
}
assert validator.allow_password is True
assert validator.allowed_schemes == set()
assert validator.allowed_hosts == set()
assert validator.allowed_ports == set()
def test_allowed_hosts_and_schemes(uri, failed_component):
"""Verify each of these fails."""
validator = (validators.Validator().allow_schemes(
"https",
"ssh",
).allow_hosts(
"github.com",
"git.openstack.org",
))
with pytest.raises(exceptions.UnpermittedComponentError) as caught_exc:
validator.validate(uri)
exc = caught_exc.value
assert exc.component_name == failed_component
def serialize(self, value):
value = six.text_type(value)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
validator = validators.Validator().require_presence_of(
'scheme').allow_schemes(*self.schemes)
try:
validator.validate(api.uri_reference(value))
except exceptions.RFC3986Exception as e:
# NOTE(erno): treat all exceptions as one for now
raise ValueError('Invalid url: {}'.format(force_text(e)))
return value
def is_valid_uri(uri,
require_scheme=True,
allowed_schemes={"http", "https"},
require_authority=True):
uri = uri_reference(uri).normalize()
validator = validators.Validator().allow_schemes(*allowed_schemes)
if require_scheme:
validator.require_presence_of("scheme")
if require_authority:
validator.require_presence_of("host")
validator.check_validity_of("scheme", "host", "port", "path", "query")
try:
validator.validate(uri)
except exceptions.ValidationError:
return False
return True
def is_uri(val: str = None) -> bool:
"""
Boolean to determine whether node content is a valid uri.
Args:
val: String value of uri
Returns: boolean
"""
is_valid = False
validator = validators.Validator().allow_schemes(
"http", "https",
"ftp").require_presence_of("scheme", "host").check_validity_of(
"scheme", "host", "path")
uri = uri_reference(val)
try:
validator.validate(uri)
is_valid = True
except (InvalidComponentsError, MissingComponentError,
UnpermittedComponentError) as ex:
logger.debug(ex)
return is_valid
def url_validator(_, attribute: attr.Attribute, urls: URLS):
if not isinstance(urls, (set, list, tuple)):
message = f'{attribute.name} is not a set, list or tuple instance: {urls}'
logger.exception(message)
raise TypeError(message)
if not all(isinstance(url, str) for url in urls):
message = f'not all items in urls are a string: {urls}'
logger.exception(message)
raise TypeError(message)
validator = validators.Validator()
allowed_schemes = ['https', 'http', 'file']
validator.allow_schemes(*allowed_schemes)
validator.check_validity_of('scheme', 'host', 'path', 'query', 'fragment')
for url in urls:
uri = iri_reference(url).encode()
try:
validator.validate(uri)
except exceptions.UnpermittedComponentError:
message = f'{url} does not have a scheme in {allowed_schemes}'
logger.exception(message)
raise ValueError(message)
except exceptions.InvalidComponentsError:
# not sure this error can happen if we use uri_reference with a string, but let's be careful
message = f'{url} is not a valid url'
logger.exception(message)
raise ValueError(message)
if uri.scheme in ('http', 'https') and uri.host is None:
message = f'url {url} must provide a host part'
logger.exception(message)
raise ValueError(message)
if uri.scheme == 'file' and uri.path is None:
message = f'url {url} must provide a path to a local file'
logger.exception(message)
raise ValueError(message)
def test_passwordless_uris_pass_validation(uri):
"""Verify password-less URLs validate properly."""
validator = validators.Validator().forbid_use_of_password()
validator.validate(uri)
def test_ensure_uri_has_a_scheme(uri):
"""Verify validation with allowed schemes."""
validator = validators.Validator().allow_schemes("https", "http")
with pytest.raises(exceptions.UnpermittedComponentError):
validator.validate(uri)
return bool(regex_http_header_field_name.search(instance))
# Do not allow rfc7230 obs-text and obs-fold in field-value for now.
# Wait and see if someone complains...
rfc7230_header_field_value = r"^[" + rfc7230_vchar + "\t]*$"
regex_http_header_field_value = re.compile(rfc7230_header_field_value)
@FormatChecker.cls_checks("http-header-field-value")
def format_check_http_header_field_value(instance):
return bool(regex_http_header_field_value.search(instance))
uri_all_components_validator = (validators.Validator().require_presence_of(
"scheme", "host", "path", "query",
"fragment").check_validity_of("scheme", "host", "path", "query",
"fragment"))
@FormatChecker.cls_checks("uri-path")
def format_check_uri_path(instance):
uri = uri_reference(
"http://example.com/{}?param=value#fragment".format(instance))
try:
uri_all_components_validator.validate(uri)
except RFC3986Exception:
return False
return (uri.scheme == "http" and uri.authority == "example.com"
and uri.path == "/" + instance and uri.query == "param=value"
and uri.fragment == "fragment")
def test_forbidden_passwords(uri):
"""Verify that passwords are disallowed."""
validator = validators.Validator().forbid_use_of_password()
with pytest.raises(exceptions.PasswordForbidden):
validator.validate(uri)
import json, logging, sys, argparse, requests
from bs4 import BeautifulSoup
from rfc3986 import validators, uri_reference
from rfc3986.exceptions import ValidationError as UriValidationError
LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(logging.StreamHandler(sys.stdout))
URI_VALIDATOR = (validators.Validator().require_presence_of(
"scheme", "host").check_validity_of("scheme", "userinfo", "host", "port",
"path", "query", "fragment"))
BASE_URL = "https://news.ycombinator.com/"
def is_valid_uri(uri):
try:
URI_VALIDATOR.validate(uri_reference(uri))
return True
except UriValidationError as exc:
LOGGER.debug(exc)
return False
def is_valid_string(string):
if not string or len(string) > 256:
return False
return True
def retrieve_title(storylink_tag):
title = storylink_tag.text
if is_valid_string(title):
def test_checking_validity_of_component():
"""Verify that we validate components we're validating."""
with pytest.raises(ValueError):
validators.Validator().check_validity_of("frob")
def test_requiring_invalid_component():
"""Verify that we validate required component names."""
with pytest.raises(ValueError):
validators.Validator().require_presence_of("frob")
def test_allowing_ports():
"""Verify the ability select ports to be allowed."""
validator = validators.Validator().allow_ports("80", "100")
assert "80" in validator.allowed_ports
assert "100" in validator.allowed_ports
def test_allowing_ports():
"""Verify the ability select ports to be allowed."""
validator = validators.Validator().allow_ports('80', '100')
assert '80' in validator.allowed_ports
assert '100' in validator.allowed_ports
import base64
from rfc3986 import validators, uri_reference
from rfc3986.exceptions import ValidationError
# only the first byte of instance id is considered, which means we can have 256 max process in the cluster.
# If there is intention of increasing the number of machines, this mask has to be changed.
INSTANCE_ID_MASK = 255
INSTANCE_ID_NUM_BITS = 8
VALID_SCHEMES = ('http', 'ftp', 'https')
validator = validators.Validator().allow_schemes(
*VALID_SCHEMES).forbid_use_of_password().require_presence_of(
'scheme',
'host').check_validity_of(*validators.Validator.COMPONENT_NAMES)
scheme_presence_validator = validators.Validator().require_presence_of(
'scheme')
def validate_url(url: str):
"""
Validate the url against several checks and raises an exception in case of validation failure
:param url: a string with the url to be validated
:return: None
"""
validator.validate(uri_reference(url))
def enforce_scheme(url: str, default_scheme: str) -> str:
def test_invalid_uri_generates_error(invalid_uri):
"""Verify we catch invalid URIs."""
uri = rfc3986.uri_reference(invalid_uri)
with pytest.raises(exceptions.InvalidComponentsError):
validators.Validator().check_validity_of("host").validate(uri)
def test_allowing_schemes():
"""Verify the ability to select schemes to be allowed."""
validator = validators.Validator().allow_schemes('http', 'https')
assert 'http' in validator.allowed_schemes
assert 'https' in validator.allowed_schemes
def test_allowing_schemes():
"""Verify the ability to select schemes to be allowed."""
validator = validators.Validator().allow_schemes("http", "https")
assert "http" in validator.allowed_schemes
assert "https" in validator.allowed_schemes
from caliper.constants import (
CALIPER_CLASSES,
CALIPER_CORE_CONTEXT,
CALIPER_CONTEXTS,
CALIPER_PROFILES,
CALIPER_PROFILES_FOR_CONTEXTS,
CALIPER_PROFILES_FOR_EVENT_TYPES,
CALIPER_PROFILE_ACTIONS,
CALIPER_TYPES,
CALIPER_TYPES_FOR_CLASSES,
EVENT_TYPES,
ENTITY_TYPES,
)
# validation regexes
_uri_validator = rfc3986_validators.Validator().require_presence_of("scheme", )
_datetime_re = re.compile(r"\A{YYYY}-{MM}-{DD}T{HH}:{mm}:{ss}.{SSS}Z\Z".format(
YYYY="([0-9]{4})",
MM="([0-9]{2})",
DD="([0-9]{2})",
HH="([0-9]{2})",
mm="([0-9]{2})",
ss="([0-9]{2})",
SSS="([0-9]{3})",
))
_uuid_urn_re = re.compile(
r"\Aurn:uuid:[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}\Z")
