def start(config_dir):
""" Start all the process
"""
websites = config.load_websites(os.path.join(config_dir, "websites.json"))
for website in websites:
webconfigs = config.load_pages(os.path.join(config_dir, website["config"]))
for webconfig in webconfigs:
if not database.check_table(webconfig["model"]):
database.initialize_table(webconfig["model"])
#get_data_link.delay(website["start_url"], website["start_model"], webconfigs)
get_data_link(website["start_url"], website["start_model"], webconfigs)
pass
def testTasks(self):
pages_config = config.load_pages("../config/skyscanner.json")
model = "skyscanner_flight"
test_page = "http://www.skyscanner.net/flights-to/lond/cheap-flights-to-london.html"
tasks.get_data_link(test_page, model, pages_config)