Python load_websites Examples

Example #1

File: task_manager.py Project: WYANG008/crawler

def start(config_dir):
    """ Start all the process
    """
    websites = config.load_websites(os.path.join(config_dir, "websites.json"))

    for website in websites:
        webconfigs = config.load_pages(os.path.join(config_dir, website["config"]))
        for webconfig in webconfigs:
            if not database.check_table(webconfig["model"]):
                database.initialize_table(webconfig["model"])
        #get_data_link.delay(website["start_url"], website["start_model"], webconfigs)
        get_data_link(website["start_url"], website["start_model"], webconfigs)
    pass

Example #2

File: test_config.py Project: WYANG008/crawler

 def testLoadWebSite(self):
     websites = config.load_websites("./websites.json")
     print len(websites)
     pass