def is_url_allowed(self, url, syntax=GYM2008):
allowed = True
# Schemes and host names are not part of the robots.txt protocol,
# so I ignore them. It is the caller's responsibility to make
# sure they match.
_, _, path, parameters, query, fragment = urllib_urlparse(url)
url = urllib_urlunparse(("", "", path, parameters, query, fragment))
url = _unquote_path(url)
if url == '/robots.txt':
return True
done = False
i = 0
while not done:
rule_type, path = self.rules[i]
if (syntax == GYM2008) and ("*" in path or path.endswith("$")):
# GYM2008-specific syntax applies here
# http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40360
if path.endswith("$"):
appendix = "$"
path = path[:-1]
else:
appendix = ""
# Multiple wildcards characters mean the same as one wildcard so they can be
# condensed into one. If I don't do this, I run the risk of creating a
# pathological regex.
# ref: https://bitbucket.org/philip_semanchuk/robotexclusionrulesparser/issues/1
path = re.sub(r'\*+', '*', path)
parts = path.split("*")
pattern = ".*".join([re.escape(p) for p in parts]) + appendix
if re.match(pattern, url):
# Ding!
done = True
allowed = (rule_type == self.ALLOW)
else:
# Wildcards are either not present or are taken literally.
if url.startswith(path):
# Ding!
done = True
allowed = (rule_type == self.ALLOW)
# A blank path means "nothing", so that effectively
# negates the value above.
# e.g. "Disallow: " means allow everything
if not path:
allowed = not allowed
i += 1
if i == len(self.rules):
done = True
return allowed
def is_url_allowed(self, url, syntax=GYM2008):
allowed = True
# Schemes and host names are not part of the robots.txt protocol,
# so I ignore them. It is the caller's responsibility to make
# sure they match.
_, _, path, parameters, query, fragment = urllib_urlparse(url)
url = urllib_urlunparse(("", "", path, parameters, query, fragment))
url = _unquote_path(url)
done = False
i = 0
while not done:
rule_type, path = self.rules[i]
if (syntax == GYM2008) and ("*" in path or path.endswith("$")):
# GYM2008-specific syntax applies here
# http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40360
if path.endswith("$"):
appendix = "$"
path = path[:-1]
else:
appendix = ""
parts = path.split("*")
pattern = "%s%s" % \
(".*".join([re.escape(p) for p in parts]), appendix)
if re.match(pattern, url):
# Ding!
done = True
allowed = (rule_type == self.ALLOW)
else:
# Wildcards are either not present or are taken literally.
if url.startswith(path):
# Ding!
done = True
allowed = (rule_type == self.ALLOW)
# A blank path means "nothing", so that effectively
# negates the value above.
# e.g. "Disallow: " means allow everything
if not path:
allowed = not allowed
i += 1
if i == len(self.rules):
done = True
return allowed
def is_url_allowed(self, url, syntax=GYM2008):
allowed = True
# Schemes and host names are not part of the robots.txt protocol,
# so I ignore them. It is the caller's responsibility to make
# sure they match.
_, _, path, parameters, query, fragment = urllib_urlparse(url)
url = urllib_urlunparse(("", "", path, parameters, query, fragment))
url = _unquote_path(url)
done = False
i = 0
while not done:
rule_type, path = self.rules[i]
if (syntax == GYM2008) and ("*" in path or path.endswith("$")):
# GYM2008-specific syntax applies here
# http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40360
if path.endswith("$"):
appendix = "$"
path = path[:-1]
else:
appendix = ""
parts = path.split("*")
pattern = "%s%s" % \
(".*".join([re.escape(p) for p in parts]), appendix)
if re.match(pattern, url):
# Ding!
done = True
allowed = (rule_type == self.ALLOW)
else:
# Wildcards are either not present or are taken literally.
if url.startswith(path):
# Ding!
done = True
allowed = (rule_type == self.ALLOW)
# A blank path means "nothing", so that effectively
# negates the value above.
# e.g. "Disallow: " means allow everything
if not path:
allowed = not allowed
i += 1
if i == len(self.rules):
done = True
return allowed