def start(config_dir):
""" Start all the process
"""
websites = config.load_websites(os.path.join(config_dir, "websites.json"))
for website in websites:
webconfigs = config.load_pages(os.path.join(config_dir, website["config"]))
for webconfig in webconfigs:
if not database.check_table(webconfig["model"]):
database.initialize_table(webconfig["model"])
#get_data_link.delay(website["start_url"], website["start_model"], webconfigs)
get_data_link(website["start_url"], website["start_model"], webconfigs)
pass
def testLoadWebSite(self):
websites = config.load_websites("./websites.json")
print len(websites)
pass