def scrape_view(request): """Renders the scrape page.""" if request.method == 'POST': form = scrape_form(request.POST) if form.is_valid(): site_choices = form.cleaned_data['site_choices_field'] scrape_choices = form.cleaned_data['scrape_choices_field'] brand_name_field = form.cleaned_data['brand_name_field'] if 'scrape' in form.data: models.scrape_li = scrape_ds.scrape_ds(site_choices, scrape_choices, brand_name_field) if not product.scrape_save(brand_name_field): form.add_form_error("Could not save scrape results") if 'retrieve' in form.data: if not product.scrape_retrieve(brand_name_field): form.add_form_error("Could not retrieve scrape results") if len(models.scrape_li) == 0: form.add_form_error("First retrieve or scrape the web for this brand") else: if 'explore' in form.data: return render(request, 'app/scraperesults.html', {'brand': brand_name_field, 'scrape_li' : models.scrape_li } ) if 'sentiment' in form.data: sentiment.sentiment(brand_name_field) if not product.scrape_save(brand_name_field): form.add_form_error("Could not save scrape results") return render(request, 'app/scraperesults.html', {'brand': brand_name_field, 'scrape_li' : models.scrape_li } ) return render(request, 'app/scrape.html', {'site' : FMI.settings.site, 'form': form, 'scrape_li' : models.scrape_li } ) else: form = scrape_form(initial={'site_choices_field':['fragrantica'],'scrape_choices_field':['accords','moods','notes']}) return render(request, 'app/scrape.html', {'site' : FMI.settings.site, 'form': form, 'year':datetime.now().year})
def crawl_product(index_choices, product): models.scrape_li = None success = False perfumes = {} designers = {} models.scrape_li = crawl_fragrantica(product) sentiment.sentiment(product) success = scrape_save(product)
def crawl_fragrantica(from_dt, brand_name, brand_variant, perfume_name): models.scrape_li = None success = False perfumes = {} designers = {} models.scrape_li = crawl_fragrantica_data(from_dt, brand_name, brand_variant, perfume_name) sentiment.sentiment(perfume_name) success = save_products_data(perfume_name) return success
def analyze(): raw_url = flask.request.args.get('url') url = urlparse(raw_url).geturl() title, text, err = scrape(url) title_sent = sentiment(title) text_sent = sentiment(text) res = { 'title': title, 'sentiment': { 'title': { 'polarity': title_sent.polarity, 'subjectivity': title_sent.subjectivity }, 'text': { 'polarity': text_sent.polarity, 'subjectivity': text_sent.subjectivity } } } return flask.jsonify(res)