def cast_uri(self, value):
if not self._type_check(value):
return False
if is_valid_uri(value, require_scheme=True):
return value
else:
raise exceptions.InvalidURI('{0} is not a valid uri'.format(value))
def test_link_value_uri(content, link=None):
"link URL must be valid"
from rfc3986 import is_valid_uri
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
assert is_valid_uri(value, require_scheme=True), \
"expected {} link '{}' to be a valid URL".format(key, value)
def test_http_link_active(content, link=None):
"link URL must be active"
import cfscrape
from requests.exceptions import RequestException
from rfc3986 import is_valid_uri, uri_reference
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
if not is_valid_uri(value, require_scheme=True):
return
parsed_value = uri_reference(value)
if parsed_value.scheme not in ("http", "https"):
return
# Hooray.
if parsed_value.host.endswith("linkedin.com"):
raise SkipTest("linkedin.com won't let us see {} anyway".format(value))
try:
r = cfscrape.create_scraper().get(value, timeout=30.0, headers={"User-Agent": USER_AGENT})
except RequestException as exc:
assert False, "error while checking {}: {}".format(value, exc)
else:
assert 200 <= r.status_code < 300, \
"expected {} link {} to be active, but got {}".format(key, value, r.status_code)
def validate(value):
if isinstance(value, six.string_types):
try:
if is_valid_uri(value):
return value
else:
raise ValueError(_("Not valid URI format: '%s'") % value)
except Exception:
raise ValueError(_("Not valid URI format: '%s'") % value)
raise ValueError(_("Expected String, got '%s'") % value)
def test_link_value_https_preferred(content, link=None):
"link URL should use HTTPS whenever possible"
from rfc3986 import is_valid_uri, uri_reference
_verify_valid_link_entry(link)
key, value = list(link.items())[0]
if is_valid_uri(value, require_scheme=True):
parsed_value = uri_reference(value)
if parsed_value.scheme == "http":
raise TestWarning("URL scheme is HTTP, but HTTPS is strongly preferred: {}".format(value))
def __call__(self, value):
if not rfc3986.is_valid_uri(value, require_scheme=True,
require_authority=True):
raise ValueError('invalid URI: %r' % value)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
self.value = value
return value
def validate(value):
if isinstance(value, six.string_types):
try:
if is_valid_uri(value):
return value
else:
raise ValueError(_("Not valid URI format: '%s'") % value)
except Exception:
raise ValueError(_("Not valid URI format: '%s'") % value)
raise ValueError(_("Expected String, got '%s'") % value)
def is_valid_uri(uri):
hostname = urlparse(uri).hostname
return (
uri
and rfc3986.is_valid_uri(
uri, 'utf-8', require_scheme=True, require_authority=True
)
and is_valid_openbazaar_scheme(uri)
and is_valid_hostname(hostname)
)
def url(url):
"""Raises an error if the url doesn't look like a URL."""
try:
if not rfc3986.is_valid_uri(url, require_scheme=True):
raise exceptions.InvalidURL(url=url)
p_url = rfc3986.urlparse(rfc3986.normalize_uri(url))
if p_url.scheme != 'http' and p_url.scheme != 'https':
raise exceptions.InvalidURL(url=url)
except Exception:
raise exceptions.InvalidURL(url=url)
return True
def cast_uri(self, value, fmt=None):
if value == '':
return value
if not isinstance(value, self.python_type):
raise exceptions.InvalidStringType('{0} is not of type {1}'.format(
value, self.python_type))
if is_valid_uri(value, require_scheme=True):
return value
else:
raise exceptions.InvalidURI('{0} is not a valid uri'.format(value))
def __call__(self, value):
if not rfc3986.is_valid_uri(value, require_scheme=True,
require_authority=True):
raise ValueError('invalid URI: %r' % value)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
if self.schemes:
scheme = rfc3986.uri_reference(value).scheme
if scheme not in self.schemes:
raise ValueError("URI scheme '%s' not in %s" %
(scheme, self.schemes))
self.value = value
return value
def ic(uriref, string, context=None, snapshoturi=None):
uri = uriref + "#" + urllib.parse.quote(string, safe="")
assert rfc3986.is_valid_uri(uri) # also rfc3986.normalize_uri
triples = [
(uri, a, uriref),
]
if snapshoturi:
triples += [
(uri, NS.po.snapshot, snapshoturi),
]
# frames = inspect.getouterframes(inspect.currentframe())
# c(outer_frame,dir(outer_frame),outer_frame.f_locals)
# outer_frame = frames[1][0]
# if "triples" in outer_frame.f_locals:
# outer_frame.f_locals["triples"]+=triples
# else:
# P.add(triples,context=context)
P.add(triples, context=context)
return uri
def cast_string(format, value, **options):
if not isinstance(value, six.string_types):
return ERROR
if format == 'uri':
if not rfc3986.is_valid_uri(value, require_scheme=True):
return ERROR
elif format == 'email':
if not re.match(_EMAIL_PATTERN, value):
return ERROR
elif format == 'uuid':
try:
uuid.UUID(value, version=4)
except Exception:
return ERROR
elif format == 'binary':
try:
base64.b64decode(value)
except Exception:
return ERROR
return value
def __call__(self, value):
if not rfc3986.is_valid_uri(
value, require_scheme=True, require_authority=True):
raise ValueError('invalid URI: %r' % value)
if self.max_length is not None and len(value) > self.max_length:
raise ValueError("Value '%s' exceeds maximum length %d" %
(value, self.max_length))
if self.schemes:
scheme = rfc3986.uri_reference(value).scheme
if scheme not in self.schemes:
raise ValueError("URI scheme '%s' not in %s" %
(scheme, self.schemes))
# NOTE(dhellmann): self.value is deprecated, and we don't want
# to trigger a deprecation warning ourselves so we modify
# self._value directly.
self._value = value
return value
def test_all_valid_inputs(self):
# tests every field returned from the app
keys_str = "[u'author', u'comments', u'points', u'rank', u'title', u'uri']"
for z in [1, POSTS_EACH_PAGE, LIMIT]:
result = subprocess.check_output([APP_NAME, "--posts", str(z)])
result = loads(result)
self.assertEquals(z, len(result))
for i in xrange(len(result)):
self.assertEquals(str(sorted(result[i].keys())), keys_str)
# author_test
self.assertTrue(isinstance(result[i]["author"], basestring))
self.assertTrue(0 < len(result[i]["author"]) < 257)
# title_test
self.assertTrue(isinstance(result[i]["title"], basestring))
self.assertTrue(0 < len(result[i]["title"]) < 257)
# uri_test
self.assertTrue(is_valid_uri(result[i]["uri"]))
# points_test comments_test rank
for c in ["points", "comments", "rank"]:
self.assertTrue(
isinstance(result[i][c], int) and result[i][c] >= 0)
def validate(self, value):
if value is None:
return
self.value = value
if self.format == "email":
if not re.match(self.EMAIL_PATTERN, value):
return False
elif self.format == "uri":
if not rfc3986.is_valid_uri(value, require_scheme=True):
return False
elif self.format == "uuid":
return self.try_convert_value(value=value,
to_type=uuid.UUID,
convertor_config={"version": 4})
elif self.format == "ipv4":
return self.try_convert_value(value=value,
to_type=ipaddress.IPv4Address)
elif self.format == "ipv6":
return self.try_convert_value(value=value,
to_type=ipaddress.IPv6Address)
elif self.format == "hostname":
if not re.match(self.HOSTNAME_PATTERN, value):
return False
elif self.format == "datetime":
self.pattern = self.field_schema.get(
"datetimePattern", defaults.FIELDS_FORMAT_DATETIME_PATTERN)
try:
datetime.datetime.strptime(value, self.pattern)
except Exception:
return False
elif self.field_schema.get("pattern",
defaults.FIELDS_TYPE_STRING_PATTERN):
self.pattern = self.field_schema.get(
"pattern", defaults.FIELDS_TYPE_STRING_PATTERN)
if not re.match(self.pattern, value):
return False
return True
def _validate_uri(instance):
return rfc3986.is_valid_uri(instance, require_scheme=True,
require_authority=True)
def validate_uri_format(entry):
return rfc3986.is_valid_uri(entry)
def translateComments(self):
triples = []
count = 0
for id_, title, body, created_at, source_id, reply_of_id, ip_address,\
author_id, referrer in self.comments_table.getMany(
("id", "title", "body", "created_at", "source_id",
"reply_of_id", "ip_address", "author_id", "referrer")):
if title and title.startswith("Teste de Stress"):
continue
if body.count(body[0]) == len(body) or\
body.lower() in ("teste", "testee", "teste 2", "texte abc")\
or "participabr teste" in body or\
(len(set(body.lower())) < 3 and len(body) > 2):
continue
commenturi = P.rdf.ic(
po.Comment, self.snapshotid+"-"+str(id_),
self.translation_graph, self.snapshoturi)
assert isinstance(created_at, datetime.datetime)
assert isinstance(body, str) and not re.findall(r"<.*>.*<.*>", body)
assert isinstance(source_id, int)
if source_id not in self.articletypes:
# articleclass = po.Article
pass
else:
# articleclass = eval(
# "po."+self.articletypes[source_id].split("::")[-1])
# triples.append((commenturi, po.type, self.articletypes[source_id].split("::")[-1])
pass
# articleuri = P.rdf.ic(
# articleclass, self.snapshotid+"-"+str(source_id),
articleuri = P.rdf.ic(
po.Article, self.snapshotid+"-"+str(source_id),
self.translation_graph, self.snapshoturi)
triples += [
(commenturi, po.createdAt, created_at),
(commenturi, po.text, body.replace("\r", "\n")),
# (commenturi, po.nChars, len(body)),
(commenturi, po.article, articleuri),
]
if title and len(title) > 2 and title.count(title[0]) != len(title)\
and not title.startswith("hub-message"):
triples += [
(commenturi, po.title, title)
]
if reply_of_id:
assert isinstance(reply_of_id, int)
commenturi0 =\
po.Comment+"#"+self.snapshotid+"-"+str(reply_of_id)
triples += [
(commenturi, po.replyTo, commenturi0)
]
if ip_address:
triples += [
(commenturi, po.ipAddress, ip_address)
]
if author_id and author_id in self.userids:
participanturi =\
po.Participant+"#"+self.snapshotid+"-"+self.userids[author_id]
triples += [
(commenturi, po.author, participanturi)
]
if referrer:
assert rfc3986.is_valid_uri(referrer)
triples += [
(commenturi, po.url, referrer)
]
count += 1
if count % 100 == 0:
c("done comments:", count)
c("finished triplification of comments")
c("ntriples:", len(triples))
P.add(triples, self.translation_graph)
c("finished add of comments")
triples = []
if triples:
c("ntriples:", len(triples))
P.add(triples, self.translation_graph)
