def scrape_view(request):
"""Renders the scrape page."""
if request.method == 'POST':
form = scrape_form(request.POST)
if form.is_valid():
site_choices = form.cleaned_data['site_choices_field']
scrape_choices = form.cleaned_data['scrape_choices_field']
brand_name_field = form.cleaned_data['brand_name_field']
if 'scrape' in form.data:
models.scrape_li = scrape_ds.scrape_ds(site_choices, scrape_choices, brand_name_field)
if not product.scrape_save(brand_name_field):
form.add_form_error("Could not save scrape results")
if 'retrieve' in form.data:
if not product.scrape_retrieve(brand_name_field):
form.add_form_error("Could not retrieve scrape results")
if len(models.scrape_li) == 0:
form.add_form_error("First retrieve or scrape the web for this brand")
else:
if 'explore' in form.data:
return render(request, 'app/scraperesults.html', {'brand': brand_name_field, 'scrape_li' : models.scrape_li } )
if 'sentiment' in form.data:
sentiment.sentiment(brand_name_field)
if not product.scrape_save(brand_name_field):
form.add_form_error("Could not save scrape results")
return render(request, 'app/scraperesults.html', {'brand': brand_name_field, 'scrape_li' : models.scrape_li } )
return render(request, 'app/scrape.html', {'site' : FMI.settings.site, 'form': form, 'scrape_li' : models.scrape_li } )
else:
form = scrape_form(initial={'site_choices_field':['fragrantica'],'scrape_choices_field':['accords','moods','notes']})
return render(request, 'app/scrape.html', {'site' : FMI.settings.site, 'form': form, 'year':datetime.now().year})
def crawl_product(index_choices, product):
models.scrape_li = None
success = False
perfumes = {}
designers = {}
models.scrape_li = crawl_fragrantica(product)
sentiment.sentiment(product)
success = scrape_save(product)
def crawl_fragrantica(from_dt, brand_name, brand_variant, perfume_name):
models.scrape_li = None
success = False
perfumes = {}
designers = {}
models.scrape_li = crawl_fragrantica_data(from_dt, brand_name,
brand_variant, perfume_name)
sentiment.sentiment(perfume_name)
success = save_products_data(perfume_name)
return success
def analyze():
raw_url = flask.request.args.get('url')
url = urlparse(raw_url).geturl()
title, text, err = scrape(url)
title_sent = sentiment(title)
text_sent = sentiment(text)
res = {
'title': title,
'sentiment': {
'title': {
'polarity': title_sent.polarity,
'subjectivity': title_sent.subjectivity
},
'text': {
'polarity': text_sent.polarity,
'subjectivity': text_sent.subjectivity
}
}
}
return flask.jsonify(res)
