def start(self):
     for url in CRAWLER_INITIAL_URLS:
         self.enqueue(resource_from_url(url))
     while (not self.q.empty() and not self.limits_reached()):
         resource = self.dequeue()
         for link in download_and_get_links(resource):
             self.visit(link)
 def visit(self, url_or_resource):
     if isinstance(url_or_resource, Resource):
         resource = url_or_resource
     else:
         resource = resource_from_url(url_or_resource)
     logging.info('visiting {}'.format(resource.url))
     if resource is None or self.is_seen(resource):
         return
     self.enqueue(resource)