def test_challenge(self):
crawler = WebCrawler(5, VerboseCrawlerLogger)
crawler.crawl("triplebyte.github.io/web-crawler-test-site/test1", None,
True)
url = "http://triplebyte.github.io/web-crawler-test-site/test1/SVG_logo.svg"
self.assertEqual(crawler.graph[url].request_type, "head")
def test_challenge(self):
crawler = WebCrawler(5, VerboseCrawlerLogger)
crawler.crawl("http://triplebyte.github.io/web-crawler-test-site/test2", None, True)
target_url = "http://triplebyte.github.io/web-crawler-test-site/test2/page2.html"
print(crawler.graph[target_url])
self.assertIsNotNone(crawler.graph[target_url])
def test_challenge(self):
crawler = WebCrawler(5, VerboseCrawlerLogger)
crawler.crawl("http://triplebyte.github.io/web-crawler-test-site/test2", None, True)
target_url = "http://triplebyte.github.io/web-crawler-test-site/test2/page2.html"
self.assertIsNotNone(crawler.graph[target_url])
def test_crawling(self):
crawler = WebCrawler(100, SilentCrawlerLogger)
crawler.crawl(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/",
None, True)
self.assert_crawled_with_get(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2",
crawler)
self.assert_crawled_with_get(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-real",
crawler)
self.assert_crawled_with_get(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake",
crawler)
self.assertEqual(
crawler.graph.nodes[
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-real"]
.status, 'success')
self.assertEqual(
crawler.graph.nodes[
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake"]
.status_code, 404)
self.assertEqual(
crawler.graph.nodes[
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake"]
.status, 'success')
self.assertIn(
"http://cdn.business2community.com/wp-content/uploads/2013/07/terrible-content.jpg",
crawler.graph.nodes)
self.assertEqual(
crawler.graph.nodes[
"http://cdn.business2community.com/wp-content/uploads/2013/07/terrible-content.jpg"]
.request_type, 'head')
def test_crawling(self):
crawler = WebCrawler(100, SilentCrawlerLogger)
crawler.crawl(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/",
None, True)
self.assertIn(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2",
crawler.graph.nodes)
self.assertIn(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-real",
crawler.graph.nodes)
self.assertIn(
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake",
crawler.graph.nodes)
self.assertEqual(
crawler.graph.nodes[
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-real"]
.status, 'success')
self.assertEqual(
crawler.graph.nodes[
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake"]
.status_code, 404)
self.assertEqual(
crawler.graph.nodes[
"http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake"]
.status, 'success')
def test_challenge(self):
crawler = WebCrawler(5, VerboseCrawlerLogger)
crawler.crawl("triplebyte.github.io/web-crawler-test-site/test1",
None,
True)
url = "http://triplebyte.github.io/web-crawler-test-site/test1/SVG_logo.svg"
self.assertEqual(crawler.graph[url].request_type, "head")
def test_challenge(self):
crawler = WebCrawler(5, VerboseCrawlerLogger)
crawler.crawl(
"http://triplebyte.github.io/web-crawler-test-site/test4/", None,
True)
self.assertTrue(
"https://triplebyte.github.io/web-crawler-test-site/test4/page3" in
crawler.graph.nodes)
def test_crawling(self):
crawler = WebCrawler(100, SilentCrawlerLogger)
crawler.crawl("http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/", None, True)
self.assertIn("http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2", crawler.graph.nodes)
self.assertIn("http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-real", crawler.graph.nodes)
self.assertIn("http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake", crawler.graph.nodes)
self.assertEqual(crawler.graph.nodes["http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-real"].status, 'success')
self.assertEqual(crawler.graph.nodes["http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake"].status_code, 404)
self.assertEqual(crawler.graph.nodes["http://triplebyte.github.io/web-crawler-test-site/already-passing-tests/page2-fake"].status, 'success')
def test_challenge(self):
# The bug here is that the crawler will hang. Don't sit around waiting
# for it to finish!
crawler = WebCrawler(5, VerboseCrawlerLogger)
crawler.crawl("http://triplebyte.github.io/web-crawler-test-site/test3/", None, True)
self.assertIn(
"http://blah.com:7091",
crawler.graph.nodes
)
def test_crawling_triplebyte(self):
crawler = WebCrawler(100, SilentCrawlerLogger)
crawler.crawl("https://www.triplebyte.com", None, True)
self.assertIn("https://www.triplebyte.com", crawler.graph.nodes)
self.assertIn("https://triplebyte.com/careers", crawler.graph.nodes)
self.assertEqual(
crawler.graph.nodes["http://www.olark.com?welcome"].request_type,
"head")
def main():
url = 'http://revistaautoesporte.globo.com/rss/ultimas/feed.xml'
crawler = WebCrawler(url)
data = crawler.build_data()
crawler.data_to_file(data)
print crawler.dump_data(data)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("target")
parser.add_argument("--number_of_threads")
parser.add_argument("--output_file")
parser.add_argument("--verbose",
help="increase output verbosity",
action="store_true")
args = parser.parse_args()
webcrawler = WebCrawler(args.number_of_threads or 5,
args.verbose and
loggers.VerboseCrawlerLogger or
loggers.SilentCrawlerLogger)
webcrawler.crawl(args.target, args.output_file)
def get_crawler(uri: str, chrome_driver_path, dump_to_local):
if parse.urlparse(uri).scheme in (
'http',
'https',
):
scraper = WebCrawler(uri, chrome_driver_path, dump_to_local)
else:
scraper = LocalCrawler(uri)
return scraper
def main():
parser = argparse.ArgumentParser()
parser.add_argument("target")
parser.add_argument("--number_of_threads")
parser.add_argument("--output_file")
parser.add_argument("--verbose",
help="increase output verbosity",
action="store_true")
args = parser.parse_args()
webcrawler = WebCrawler(
args.number_of_threads or 5,
args.verbose and loggers.VerboseCrawlerLogger
or loggers.SilentCrawlerLogger)
webcrawler.crawl(args.target, args.output_file)
def get_one_day(self):
params = {
'q': self.city,
'appid': self.api_key,
'lang': self.lang,
'units': self.units
}
uri = self.uri + "?" + urllib.urlencode(params, True)
print uri
data = WebCrawler.get_data(uri)
print(data)
self.decode_json(data)
return self.speech_string
class TestWebCrawler(unittest.TestCase):
def setUp(self) -> None:
self.content_fetcher = unittest.mock.Mock()
self.content_fetcher.retrieve_page.return_value = self.generate_mock_page(
)
self.web_crawler = WebCrawler(self.content_fetcher)
def generate_mock_page(self):
return "<!DOCTYPE html>" \
"<html><body><h1>Some header</h1><p>Some text</p>" \
"<a href='http://some_link2.com'></a>" \
"<a href='http://some_link2.com'></a>" \
"</body></html>"
def test_crawl_does_not_return_duplicate_links(self):
urls = self.web_crawler.discover("http://some_link.com", limit=10)
self.assertEqual(urls,
["http://some_link.com", "http://some_link2.com"])
def test_crawl_does_not_give_more_links_than_the_limit(self):
urls = self.web_crawler.discover("http://some_link.com", limit=1)
self.assertEqual(urls, ["http://some_link.com"])
def __init__(self, outfile, startUrl, limit, searchType, keyword):
self.outfile = outfile
self.startPage = startUrl
self.limit = limit
if keyword is None:
self.keywordExists = False
else:
self.keywordExists = True
self.keyword = keyword
self.searchType = searchType
self.currentLevel = 0
self.webCrawler = WebCrawler(keyword)
self.idCount = -1
# 0 represents root level
self.rootNode = PageNode(None, self.getUID(), startUrl, 0)
self.activeNode = None
self.rootError = None
self.crawled = set()
# seed the random integer generator for DFS method
random.seed()
def setUp(self) -> None:
self.content_fetcher = unittest.mock.Mock()
self.content_fetcher.retrieve_page.return_value = self.generate_mock_page(
)
self.web_crawler = WebCrawler(self.content_fetcher)
from parse import Parse
from webcrawler import WebCrawler
from interface import Interface
if __name__ == "__main__":
interface = Interface()
parse = Parse()
args = parse.get_parse()
parse.do_parse(args)
webcrawler = WebCrawler(parse)
webcrawler.get_headers(interface.header_inter())
webcrawler.get_data(interface.data_inter())
webcrawler.get_url(interface.url_inter())
webcrawler.do_crawl()
############################################
# Parser for mystuwe.de #
############################################
from webcrawler import WebCrawler
from stuweparser import StuweParser
from datetime import *
morgenstelle = "http://www.my-stuwe.de/mensa/mensa-morgenstelle-tuebingen"
wilhelm = "http://www.my-stuwe.de/mensa/mensa-wilhelmstrasse-tuebingen/"
alldaysWillhelm = "http://www.my-stuwe.de/mensa/mensa-wilhelmstrasse-tuebingen/?woche="+str(datetime.today().isocalendar()[1] + 1)
alldays = "http://www.my-stuwe.de/mensa/mensa-morgenstelle-tuebingen/?woche="+str(datetime.today().isocalendar()[1] + 1)
print("Crawling: " + alldays)
crawler = WebCrawler(alldays)
print("Crawling: " + alldaysWillhelm)
crawler2 = WebCrawler(alldaysWillhelm)
print("Start xml generation")
parser = StuweParser(crawler.getHTML())
parser2 = StuweParser(crawler2.getHTML())
try:
#parser.generateXML()
parser.generateWeekXML("overviewMorgen.xml")
parser2.generateWeekXML("overviewWillhelm.xml")
print("XML generated")
except Exception as e:
print("An error occurred while generating xml file")
print(e)
from webcrawler import WebCrawler
import sys
firefox = {'User-Agent':'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0'}
try:
user_input = sys.argv[1]
except IndexError:
user_input = False
crawler = WebCrawler(user_input, firefox)
#crawler.validate_url()
crawler.get_amazon_image()
#!/usr/bin/python
import sys
from webcrawler import WebCrawler
if __name__ == "__main__":
website = 'https://pier31.co'
if len(sys.argv) <= 1:
print "\nYou didn't enter an address. Defaulting to %s" % website
else:
website = sys.argv[1]
print "\nChoosen address: %s" % website
web_crawler = WebCrawler(website)
web_crawler.crawl_it()