def index(): form = UrlEntry() if form.validate_on_submit(): url_to_prospect = format_url(form.url.data) domain_data = models.DomainData.query.filter_by(domain_url=url_to_prospect).first() crawler = Crawler() if domain_data: domain_data = crawler.scrape_domain_data(url_to_prospect, domain_data) else: domain_data = crawler.scrape_domain_data(url_to_prospect) pages_to_scrape = crawler.spider_site(domain_data.domain_url) pages_data = [crawler.scrape_page_data(page_to_scrape, domain_data) for page_to_scrape in pages_to_scrape] ranker = Ranker() domain_data.ranking = ranker.rank_site(domain_data) domain_data.level = ranker.domain_level_calculator(domain_data.ranking) db.session.add(domain_data) db.session.add_all(pages_data) db.session.commit() return redirect(url_for('siteinspect', site_name=domain_data.site_name)) return render_template("index.html", form=form)