def links_test():
agent = CAgent("rit.edu", 443)
agent.uri = "/computing/directory?term_node_tid_depth=4919"
page = agent.request()
links = get_addresses(page)
for link in links:
print(link)
def agent_wrapper(url, scope):
try:
port, host, uri = break_url(url)
sub = {url}
if sub.issubset(visited):
return set(), set()
agent = CAgent(host, port)
agent.uri = uri
agent.scope = scope
page = ""
page = agent.request()
visited.add(url)
lset = get_addresses(page)
eset = get_emails(page)
return lset, eset
except Exception as e:
return set(), set()