def is_url_allowed(url):
"""
Returns ``True`` if robots.txt rules for given URL allow fetching it. This
function parses the robots rules for given URL (if any) and returns a
boolean flag that tells you whether fetching it is allowed. Note that it
doesn't test whether the URL exists on the host.
:param url: URL to test
:returns: ``True`` if URL can be fetched, ``False`` otherwise
"""
robots = RobotParser()
robots.user_agent = UA_STRING
robots.fetch(get_robots_url(url))
if robots.response_code != 200:
return True
return robots.is_allowed(UA_STRING, url)
