-
Notifications
You must be signed in to change notification settings - Fork 0
/
server.py
98 lines (84 loc) · 3.05 KB
/
server.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
"""
REST API for generating sentences.
"""
import logging
import psycopg2
import threading
import os
from flask import Flask, jsonify, request
from logging.handlers import RotatingFileHandler
from scraper import Scraper
from sentence_generator import SentenceGenerator
from sentence_classifier import SentenceClassifier
app = Flask(__name__)
generate = None
SCRAPING_INTERVAL = 3600
@app.route("/sentence")
def generate_sentence():
"""
Generates a sentence and sends a JSON response in the form:
{sentence : "foo bar."}.
"""
sentence = generate.generate_sentence(initial_word="I")
app.logger.info("Generated sentence: %s" % sentence)
return jsonify(sentence=sentence)
@app.route("/sentence", methods=['POST'])
def put_sentence():
"""
Puts the classified sentence into the database.
"""
sentence = request.form['sentence']
was_funny = request.form['wasFunny']
app.logger.info("Received sentence %s with funny = %s" % (sentence, was_funny))
response = insert_sentence_into_db(sentence, was_funny)
return jsonify(response=response)
def insert_sentence_into_db(sentence, funny):
"""
Inserts a sentence into the database of sentences. The param,
'sentence' is the sentence to be inserted, 'funny' is whether or not
the sentence was funny or not.
"""
conn = psycopg2.connect(database=os.environ["DATABASE"], user=os.environ["USER"])
cur = conn.cursor()
app.logger.debug("Attempting to insert %s..." % sentence)
sentence = sentence.replace("'", "''") # escape quotes
sql_string = "INSERT INTO funny_sentences (sentence, funny) VALUES ('%s', '%s')" \
% (sentence, funny)
try:
msg = "Successfully inserted %s" % sentence
cur.execute(sql_string)
app.logger.debug(msg)
conn.commit()
except psycopg2.IntegrityError:
msg = "The phrase '%s' could not be inserted into the database" % sentence
app.logger.debug(msg)
conn.rollback()
return msg
def setup_logger():
"""
Sets up the logger to info level as well as setting up the log file.
"""
formatter = logging.Formatter("[%(asctime)s] {%(pathname)s%(lineno)d} %(message)s")
handler = RotatingFileHandler("server.log", maxBytes=10000, backupCount=1)
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
app.logger.addHandler(handler)
app.logger.setLevel(logging.INFO)
def scrape():
"""
Sets up the scraper to scrape HN and Reddit.
"""
app.logger.info("Scraping Reddit")
scrape_reddit = Scraper(app.logger)
scrape_reddit.gather_reddit_data()
app.logger.info("Finished gathering data, inserting into DB")
scrape_reddit.insert_into_db()
app.logger.info("Finished inserting into DB, sleeping for %d minutes..." % \
(SCRAPING_INTERVAL / 60.0))
threading.Timer(SCRAPING_INTERVAL, scrape).start()
if __name__ == "__main__":
setup_logger()
classifier = SentenceClassifier(app.logger)
generate = SentenceGenerator(classifier, app.logger)
scrape()
app.run(port=int(os.environ["FLASK_PORT"]))