def review_summarization(movie_id):
try:
positive_reviews = Reviews.objects(Movie_id=movie_id, Sense='positive').order_by('-Polarity')
negative_reviews = Reviews.objects(Movie_id=movie_id, Sense='negative').order_by('Polarity')
neutral_reviews = Reviews.objects(Movie_id=movie_id, Sense='neutral')
positive_string=""
negative_string=""
neutral_string=""
for review in positive_reviews:
positive_string+=(review.Body+" ")
for review in negative_reviews:
negative_string+=(review.Body+" ")
for review in neutral_reviews:
neutral_string+=(review.Body+" ")
positive_summ = text_summarization.summarize(text=positive_string)
negative_summ = text_summarization.summarize(text=negative_string)
neutral_summ = text_summarization.summarize(text=neutral_string)
return {
"positive_key_reviews":positive_summ,
"negative_key_reviews":negative_summ,
"neutral_key_reviews":neutral_summ
}
except:
raise InvalidQueryError("Summary cannot be generated")
def reviews(request, template='review/reviews.html', extra_context=None):
review_positiv = Reviews.objects(
review_eval='positiv').order_by('+hotel_name')
review_neutral = Reviews.objects(
review_eval='neutral').order_by('+hotel_name')
review_negativ = Reviews.objects(
review_eval='negativ').order_by('+hotel_name')
best_hotels = Reviews.objects.aggregate(
{"$match": {
"review_eval": "positiv"
}}, {"$group": {
"_id": "$hotel_name",
"sum": {
"$sum": 1
}
}}, {"$sort": {
"sum": -1
}}, {"$limit": 10})
context = {
# 'reviews':review_list,
#Future work
'reviews_positiv': review_positiv,
'reviews_neutral': review_neutral,
'reviews_negativ': review_negativ,
'best_hotels': best_hotels
}
if extra_context is not None:
context.update(extra_context)
return render(request, 'review/reviews.html', context)
def review_summarization(movie_id):
try:
positive_reviews = Reviews.objects(
Movie_id=movie_id, Sense='positive').order_by('-Polarity')
negative_reviews = Reviews.objects(
Movie_id=movie_id, Sense='negative').order_by('Polarity')
neutral_reviews = Reviews.objects(Movie_id=movie_id, Sense='neutral')
positive_string = ""
negative_string = ""
neutral_string = ""
for review in positive_reviews:
positive_string += (review.Body + " ")
for review in negative_reviews:
negative_string += (review.Body + " ")
for review in neutral_reviews:
neutral_string += (review.Body + " ")
positive_summ = text_summarization.summarize(text=positive_string)
negative_summ = text_summarization.summarize(text=negative_string)
neutral_summ = text_summarization.summarize(text=neutral_string)
return {
"positive_key_reviews": positive_summ,
"negative_key_reviews": negative_summ,
"neutral_key_reviews": neutral_summ
}
except:
raise InvalidQueryError("Summary cannot be generated")
def create_recipe_review(payload, recipe_id):
body = request.get_json()
review = body.get('review', None)
recipe = Recipe.query.filter(Recipe.id == recipe_id).one_or_none()
print(recipe)
if (review == None or recipe == None):
abort(404)
try:
review = Reviews(recipe_id=recipe_id, review=review)
review.insert()
return jsonify({'success': True})
except:
abort(422)
def update_movie_review(review_id, movie_id=None, body=None, timestamp=None, deleted=None, review_author=None):
try:
review = Reviews.objects(id=review_id)
if movie_id:
review.update(Movie_id=movie_id)
if body:
review.update(Body=body)
analysis = comments_analysis.sentiment(sentence=body)
review.update(Polarity=analysis['polarity'])
review.update(Sense=analysis['sense'])
if timestamp:
review.update(Timestamp=timestamp)
if deleted:
review.update(Deleted=deleted)
if review_author:
review.update(Review_author=review_author)
return {'Success':True}
except:
raise InvalidQueryError("Check for details")
def test_9_Reviews_delete(self):
'''
Testing Deletion of Records on Reviews
'''
new_r = Reviews(
u"0010", # rest id
u"Little_Italy", # name
u"italy", # yelp id
u"italian",
u"Italian",
u"12/1/2014", # date
4, # rating
u"pebs", # username
u"pretty good", # review
u"/review_profiles/pebs", # img url
u"/review/1", # rev url
u"002", # zip
)
self.session_token.add(new_r)
self.session_token.commit()
r = self.session_token.query(Reviews).filter_by(
restaurant_id=u"0010").first()
self.session_token.delete(r)
self.session_token.commit()
r = self.session_token.query(Reviews).filter_by(
restaurant_id=u"0010").first()
assert r is None
print("Passed Test9")
def update_movie_review(review_id,
movie_id=None,
body=None,
timestamp=None,
deleted=None,
review_author=None):
try:
review = Reviews.objects(id=review_id)
if movie_id:
review.update(Movie_id=movie_id)
if body:
review.update(Body=body)
analysis = comments_analysis.sentiment(sentence=body)
review.update(Polarity=analysis['polarity'])
review.update(Sense=analysis['sense'])
if timestamp:
review.update(Timestamp=timestamp)
if deleted:
review.update(Deleted=deleted)
if review_author:
review.update(Review_author=review_author)
return {'Success': True}
except:
raise InvalidQueryError("Check for details")
def comment_it():
"""This adds a comment to the database for the specific video"""
info = request.json
video_title = info["videoTitle"][:100]
success = current_user.is_authenticated
if request.method == "POST" and success:
record = Reviews(
video_title=video_title,
user_id=current_user.id,
review=info["comment"][:280],
video_id=info["videoId"],
)
db.session.add(record)
db.session.commit()
comments = Reviews.query.filter_by(video_id=info["videoId"]).all()
comment_list = []
for comment in comments:
comment_list.append({
"user":
User.query.filter_by(id=comment.user_id).first().user_name,
"text":
comment.review,
})
return jsonify({"comment_list": comment_list})
def add_book(form):
title = form.get('title')
author = form.get('author')
isbn = form.get('isbn')
book = Books(title=title, author=author, isbn=isbn)
year = form.get('year')
if year:
year = int(year)
book.year = year
review = form.get('review')
rating = form.get('rating')
try:
db.session.add(book)
db.session.commit()
except Exception as e:
print(e)
return False
book = Books.query.filter_by(title=title).first()
print('Book', book)
if rating:
rating = int(form.get('rating'))
book.num_rating = 1
book.avg_rating = rating
review = Reviews(rating=rating, text=review, bookid=book.id)
try:
db.session.add(book)
db.session.add(review)
db.session.commit()
except:
return False
return True
def insertDB():
foler_path = r'C:/Users/lanti/Desktop/JDReviewAnalyse/PythonCrawler/Data/'
folders = os.listdir(foler_path)
with open(r'D:/Courses/zhuanyeshixun2/breakpoint.txt', 'r') as f:
breakpoint = int(f.read())
for i in range(breakpoint, len(folders)):
id = folders[i]
print str(i) + '\n'
with open(r'D:/Courses/zhuanyeshixun2/breakpoint.txt', 'w') as f:
f.write(str(i))
with codecs.open(foler_path + id + '/basicInfo.txt',
'r', encoding='utf-8') as f:
info = f.read().split("--------------------------------------------------------------------------------------------------------")
# try:
productName = info[0].strip()
price = float(info[1].strip())
productConfig = info[2]
# except Exception, e:
# continue
product = Product(id = id, information = productConfig, name = productName, price = price)
db.session.add(product)
db.session.commit()
with codecs.open(foler_path + id + '/reviews.txt', 'r', encoding='utf-8') as f:
reviews = f.read().split("-------------------------------------------------------------------------------------------------------------------------------------------------------------\r\n")
for review in reviews:
data = review.split("\r\n")
if data[0] == "":
break
try:
content = data[0] # Review content
productId = id
# print "productId: " + productId + '\n'
score = int(data[2].split(':')[1]) # Review score
# print "score: " + str(score) + '\n'
creationTime = data[3].split(':')[1] # Review creationTime
# print "creationTime: " + creationTime + '\n'
userfulCount = int(data[4].split(':')[1]) # Review usefulCount
# print "userfulCount: " + str(userfulCount) + '\n'
expValue = int(data[5].split(':')[1]) # Review expValue
# print 'expValue: ' + str(expValue) + '\n'
images = int(data[6].split(':')[1]) # Review images
# print "images: " + str(images) + '\n'
length = int(data[7].split(':')[1]) # Review length
# print "length: " + str(length) + '\n'
count = int(data[8].split(':')[1]) # Review reply count
# print "count: " + str(count) + '\n'
afterComment = int(data[9].split(':')[1]) # IsAfterComment
# print "afterComment: " + str(afterComment) + '\n'
productType = data[10].split(':')[1] # Review productType
# print "productType: " + productType + '\n'
# print '------------------------------------------------------------------------------------------------\n'
except Exception, e:
continue
comment = Reviews(product_id=productId, content=content, score=score, time=creationTime,
usefulVoteCount=userfulCount,
userExpValue=expValue, images=images, length=length, replyCount=count,
afterComment=afterComment, product_type=productType)
db.session.add(comment)
db.session.commit()
def reviewDetails():
reviews = Reviews.query.all()
form = ReviewForm(request.form)
if request.method == 'POST':
if form.validate():
if form.id.data == '':
r = Reviews(form.doctorID.data, form.description.data, form.username.data)
db_session.add(r)
db_session.commit()
else:
r = Reviews.query.get(int(form.id.data))
r.doctorID = form.doctorID.data
r.description = form.description.data
r.username = form.username.data
db_session.commit()
reviews = Reviews.query.all()
return render_template('review.html', reviews=reviews)
def update_vote_average_count(id_movie, user_review, vote):
vote = float(vote)
get_vote = Movies.select(
Movies.vote_average).where(Movies.id == id_movie).limit(1).get()
get_average = Movies.select(
Movies.vote_count).where(Movies.id == id_movie).limit(1).get()
new_vote = ((get_vote.vote_average * get_average.vote_count) +
1) / (get_average.vote_count + 1)
updating_vote = Movies.update(vote_average=new_vote).where(
Movies.id == id_movie)
updating_vote.execute()
Movies.update(vote_count=get_average.vote_count +
1).where(Movies.id == id_movie).execute()
Reviews.create(review=user_review,
score=vote,
user_id=session['id'],
movie_id=id_movie)
return 'Done'
def create_review(movie_id):
movie = Movies.query.get(movie_id)
if request.method == "POST":
p = review_params()
p['user_id'] = current_user.id
review = Reviews(**p)
review.text = bleach.clean(request.form.get('text'))
db.session.add(review)
db.session.commit()
movie.rating_num = movie.rating_num+1
movie.rating_sum = movie.rating_sum+int(request.form.get('rating'))
db.session.add(movie)
db.session.commit()
flash("Рецензия успешно оставлена", "success")
return redirect(url_for('processes.read', movie_id=movie_id))
return render_template('reviews/create_review.html', movie_id=movie_id)
def verify_review(form, book):
rating = form.get('rating')
review = form.get('review')
review = Reviews(text=review, bookid=book.id)
if rating:
rating = int(rating)
review.rating = rating
try:
db.session.add(review)
db.session.commit()
if rating:
book.avg_rating = ((book.num_rating * book.avg_rating) +
rating) / (book.num_rating + 1)
book.num_rating += 1
db.session.add(book)
db.session.commit()
except Exception as e:
print(e)
return False
return True
def comment():
name1 = request.form.get("comment")
isbn = request.form.get("isbn")
book = Books.query.filter_by(isbn=isbn).first()
review = Reviews(username=current_user.username, title=book.title,
year=book.year, isbn=book.isbn, author=book.author, comment=name1)
db.session.add(review)
db.session.commit()
flash('Comment Posted', 'sucess')
return redirect(url_for('account'))
def book_details(id):
book = Books.query.get(id)
page_id = book.id
page_isbn = book.isbn
res = requests.get("https://www.goodreads.com/book/review_counts.json", params={"key": "HMOBPhAT2PnNTFaV4BiqEw", "isbns":page_isbn.strip()})
if res.status_code == 200:
data = res.json()
print(data.keys())
for k in data['books']:
goodreads_rating = k['average_rating']
ratings_count = k['ratings_count']
else:
goodreads_rating = "Not available"
ratings_count = "Not available"
form = SubmitReview()
book_reviews = book.reviews
func = fetch_user_comment
if (current_user.id in [b.users.id for b in book_reviews]) is False:
if form.validate_on_submit():
comment = form.comment.data
rating = int(form.rating.data)
review = Reviews(books_id=page_id, comment=comment,
users_id=current_user.id, rating=rating)
session.add(review)
session.commit()
return "succesful"
else:
flash("one of the fields is empty")
else:
flash("you already made a comment")
rating_list = []
for b in book_reviews:
rating_list.append(b.rating)
if len(rating_list) == 0:
ave_rating = "no rating yet"
else:
ave_rating = sum(rating_list)/len(rating_list)
no_of_comments = len(rating_list)
my_api = [{'title': book.title}, {'author': book.author }, {'year': book.year},
{'isbn': page_isbn}, {'review_count': no_of_comments}, {'average_score': ave_rating}]
print(my_api)
return render_template('reviews.html', book_details=book_details,
func=func, book_reviews=book_reviews, book=book,
form=form, page_id=page_id, ave_rating=ave_rating,
goodreads_rating=goodreads_rating, ratings_count=ratings_count)
def review(movie_id):
movie = get_reviews(movie_id)
message = ""
if request.method == 'POST':
name = request.form.get('name')
review_txt = request.form.get('review')
if name != '' or review_txt != '':
try:
df = pd.DataFrame({"review": [review_txt]})
# Clean Data
content = df['review'].apply(lambda x: clean_text(x))
print(content)
# Import Model
model = tf.keras.models.load_model('movie_review_model')
# Preparing the tokenizer
tokenizer = Tokenizer(num_words=100)
encoded_text = tokenizer.texts_to_sequences([content[0]])
max_length = 2
padded_text = pad_sequences(encoded_text,
maxlen=max_length,
padding='post')
# Predict
prediction = model.predict(padded_text)
except Exception as e:
message = "Operation Failed! Please try again"
print(int(prediction[0]))
book = Reviews(reviewer_name=name,
review=review_txt,
rating=int(prediction[0]),
movie_id=movie_id)
failed = False
db.session.add(book)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
db.session.flush() # for resetting non-commited .add()
failed = True
if failed:
message = "Operation Failed! Please try again"
else:
message = "Your review received a " + str(
prediction[0]) + " Star rating"
else:
message = "Please enter a name and review"
return render_template("review.html", movie=movie, message=message)
def add_review(film_id):
if request.method == 'POST':
# Getting review data from form
form = request.form
session = Session(engine)
# Creating query
new_review = Reviews(film_id=film_id,
user_id=current_user.id,
score=form['score'],
text=form['text'])
session.add(new_review)
# Saving new review
session.commit()
return redirect(url_for('view_film', film_id=film_id))
return render_template('films/add_review.html')
def add_review():
review = request.get_json()['review']
book = request.get_json()['book']
user = Users.objects(username=session['user']).only(
'username', 'profile_pic').get()
book = Books.objects(id=book).get()
try:
book.reviews.get(username=session['user'])['review_text'] = review
except DoesNotExist:
book.reviews.append(
Reviews(username=user['username'],
profile_pic=user['profile_pic'],
review_text=review))
book.save()
return jsonify({"result": True})
def rate_book():
book_id = request.get_json()['book']
rating = request.get_json()['rating']
book = Books.objects(id=book_id).get()
try:
book.reviews.get(username=session['user'])['rating'] = rating
book.reviews.get(
username=session['user'])['created'] = datetime.utcnow()
except DoesNotExist:
user = Users.objects(username=session['user']).only(
'username', 'profile_pic').get()
book.reviews.append(
Reviews(username=user['username'],
profile_pic=user['profile_pic']))
book.save()
return jsonify({"result": True})
def splitAttributes(self):
i = 1
comments = []
for review in self.getReviews:
for one in review:
try:
data = one.split('\r\n')
content = data[0] # Review content
print data[1].split(':')[1].strip()
score = int(data[1].split(':')[1].strip()) # Review score
creationTime = data[2].split(':')[1] # Review creationTime
userfulCount = int(
data[3].split(':')[1].strip()) # Review usefulCount
expValue = int(
data[4].split(':')[1].strip()) # Review expValue
images = int(
data[5].split(':')[1].strip()) # Review images
length = int(
data[6].split(':')[1].strip()) # Review length
count = int(
data[7].split(':')[1].strip()) # Review reply count
afterComment = int(
data[8].split(':')[1].strip()) # IsAfterComment
productType = data[9].split(
':')[1].strip() # Review productType
comment = Reviews(product_id='4130043',
content=content,
score=score,
time=creationTime,
usefulVoteCount=userfulCount,
userExpValue=expValue,
images=images,
length=length,
replyCount=count,
afterComment=afterComment,
product_type=productType) #breakpoint
comments.append(comment)
except IndexError, e:
continue
i += 1
return comments
def test_8_Reviews__manual_integrity(self):
'''
Testing query data on Reviews
'''
new_r = Reviews(
u"009", # rest id
u"Little_Italy", # name
u"italy", # yelp id
u"italian",
u"Italian",
u"12/1/2014", # date
4, # rating
u"pebs", # username
u"pretty good", # review
u"/review_profiles/pebs", # img url
u"/review/1", # rev url
u"002", # zip
)
self.session_token.add(new_r)
self.session_token.commit()
rev = self.session_token.query(Reviews).filter_by(
restaurant_id=u"009").first()
assert not self.session_token.query(Reviews) is None
# test serialize
assert isinstance(rev.to_dict(), dict)
# test query
assert rev.date == u"12/1/2014"
assert rev.rating == 4
assert rev.username == u"pebs"
assert rev.profile_picture_url == u"/review_profiles/pebs"
assert rev.review_url == u"/review/1"
assert rev.restaurant_id == u"009"
print("Passed Test8")
def get_movie_reviews(movie_id,offset=0,limit=10):
positive_reviews=[]
negative_reviews=[]
neutral_reviews=[]
reviews = Reviews.objects(Movie_id=movie_id)
#print reviews
for review in reviews:
d={}
d['sentence']=review.Body
d['polarity']=review.Polarity
if review.Sense == 'positive':
positive_reviews.append(d)
elif review.Sense == 'neutral':
neutral_reviews.append(d)
else:
negative_reviews.append(d)
positive_reviews = sorted(positive_reviews, key=lambda k: k['polarity'], reverse=True)
negative_reviews = sorted(negative_reviews, key=lambda k: k['polarity'], reverse=True)
#rating = calculate_rating(positive_reviews,negative_reviews)
#print rating
return positive_reviews,neutral_reviews,negative_reviews
def load_reviews_to_db(restaurant):
search_url = search_base_url + restaurant + search_suffix
review_objs_list, biz_id = grab_review_objects(search_url, restaurant)
for review in review_objs_list:
review_id = review.attrs['data-review-id']
date = review.find('meta',
attrs={
'itemprop': 'datePublished',
'content': True
}).attrs['content']
date = datetime.strptime(date, "%Y-%m-%d")
rating = float(
review.find('meta', {
'itemprop': 'ratingValue',
'content': True
}).attrs['content'])
location = review.find('li', attrs={
'class': 'user-location'
}).get_text().replace("\n", "")
text = re.sub(ur'\u00a0', '',
review.find('p', attrs={
'itemprop': 'description'
}).get_text(), re.UNICODE) # get rid of unicode
r_obj = Restaurants.query.filter_by(biz_id=biz_id).first()
r_name = r_obj.common_name
temp_review_obj = Reviews(review_id=review_id,
restaurant_name=r_name,
biz_id=biz_id,
rating=rating,
text=text,
location=location,
date=date)
db.session.add(temp_review_obj)
db.session.commit()
print "reviews added!"
def new_review(user):
body = request.get_json()
print(body,user)
id_users = str(user["id"])+"@"+str(body["id"])
new_review = Reviews(id_traveler=user["id"], id_pro=body["id"],id_users=id_users, value=body["value"])
userpro = Userpro.query.filter_by(id=body["id"]).first()
old_total = userpro.total_reviews
new_total = old_total+1
old_sum = userpro.sum_reviews
new_sum = old_sum+body["value"]
old_percent = userpro.percent_reviews
new_percent = round(new_sum/new_total,1)
userpro.total_reviews = new_total
userpro.sum_reviews = new_sum
userpro.percent_reviews = new_percent
db.session.add(new_review)
db.session.add(userpro)
db.session.commit()
return jsonify("review added"), 200
def addPerf(play_id):
# to avoid error if play_id is bigger than 39 in the uri
if play_id >= 40:
return redirect('/')
# redirect to non-editable page if not logged-in
if not loggedIn():
return redirect(url_for('showPlayPerf', play_id=play_id))
login_session['current_uri'] = url_for('addPerf', play_id=play_id)
play = Plays.query.filter_by(id=play_id).first()
theatres = Theatres.query.order_by(Theatres.theatre_name).all()
# on submitting the form create the new entries in the database
if flask_req.method == 'POST':
# creating a new Review entry in the DB
p_date = flask_req.form['p_date']
newReview = Reviews(review_title=flask_req.form['review_title'],
performance_date=dateTr(p_date, 'y-m-d'),
review_link=flask_req.form['review_link'],
user_id=current_user.id)
db.session.add(newReview)
db.session.commit()
# creating a new Performance entry in the DB
newPerformance = Performances(play_id=play_id,
theatre_id=flask_req.form['theatre'],
review_id=newReview.id,
user_id=current_user.id)
db.session.add(newPerformance)
db.session.commit()
return redirect(url_for('modifyPlayPerf', play_id=play_id))
return render_template('form_perform.html', play=play, theatres=theatres)
def update_movie_review(movie_id, sentence, review_author='Pankaj Gupta'):
try:
analysis = TextBlob(sentence)
polarity = list(analysis.sentiment)[0]
if(polarity>0):
review = Reviews(Movie_id=movie_id, Body=sentence, Polarity=(polarity*100), Sense='positive', Review_author=review_author)
review.save()
elif(polarity<0):
review = Reviews(Movie_id=movie_id, Body=sentence, Polarity=(-1*polarity*100), Sense='negative', Review_author=review_author)
review.save()
else:
review = Reviews(Movie_id=movie_id, Body=sentence, Polarity=polarity, Sense='neutral', Review_author=review_author)
review.save()
print('Successfully updated')
return {'Success': True}
except:
raise InvalidQueryError("Check for details")
def process_item(self, item, spider):
review = Reviews(**item)
self.add_item_to_database(review)
return item
def leave_review(current_user, json_data):
"""
Leave a review for a single place, the reviewer needs to be authed
:param json_data: review scores, auth token
:return: Success or Error based on factors
"""
# This can be a Google Place Refernce ID, or our database primary key
place_id = json_data['id']
# Try to find the place in our Database, if we can't,
# call Google Places API to get the details to add to our database
find_it = whos_data(place_id)
# If we don't have the data we need to add it, we do that in this code block / if we have it this skips
if find_it is False:
newly_added = add_place(place_id)
# Now lets get the details from the review
cat1 = json_data['cat1']
cat2 = json_data['cat2']
cat3 = json_data['cat3']
cat4 = json_data['cat4']
cat5 = json_data['cat5']
comments = json_data['comments']
visited_date = json_data['visited']
# Make sure all of our categories has valid data, if not we need to return an error
valid_data = [cat1, cat2, cat3, cat4, cat5]
# 0 is form default, if 0 something is missing
if 0 in valid_data:
return {'Err': 'It looks likes the review is missing some required data!'}, 401
if visited_date:
# Make a datetime object from the user input
# data should look like '2018-01-10T05:00:00.000Z'
# TODO this needs refactor both front and back ends
# Get rid of excess string
date = visited_date[0:10]
date_obj = datetime.strptime(date, '%Y-%m-%d')
else:
date_obj = None
# TODO add something to see if any fields are blank
# Set the Avg Rating
review_avg = (int(cat1) + int(cat2) + int(cat3) + int(cat4) + int(cat5)) / 5
# Get the correct foreign key to use
if find_it is False:
fkey_place = newly_added.id
else:
fkey_place = find_it.id
add_new_review = Reviews(place_id=fkey_place,
reviewer_id=current_user.id,
visited_date=date_obj,
cat_1=cat1,
cat_2=cat2,
cat_3=cat3,
cat_4=cat4,
cat_5=cat5,
review_avg=str(review_avg), # store as a string, later can take it into float
comments=comments)
# TODO add try/except incase of error
db.session.add(add_new_review)
db.session.commit()
# TODO add the factors
return {'Success': 'Thanks for reviewing this location!', 'place_id': fkey_place}, 200