def __init__(self, faq_file): self.verbose_level = 1 self.make_dirs() base_name = os.path.basename(faq_file) self.nn = FastNeuralNet(db_path + base_name + "_nn.db") # Not crawling data, from Crawler we are just using a couple of methods # to get word IDs and URL IDs self.crawler = Crawler(db_path + base_name + "_index.db") self.set_faq_file(faq_file) self.sp = None
from searchengine import Crawler pagelist = ['http://kiwitobes.com/wiki/Perl.html'] crawler = Crawler() Crawler.crawl(pagelist)