def build_psq_classifier(end_date_str=None):
"""Build a predictor of whether or not a question will be closed as
homework / for 'lack of context'. This is accomplished by building a
linear SVC model, trained on old post data.
If end_date_str isn't specified, it is initialized to two weeks prior.
Pickles the classifier, an instance of sklearn.svm.LinearSVC. Also stores
some basic data metrics.
Note that we only use posts written after 2013-06-25, the date on which
the first such closure reason was instituted.
"""
if end_date_str == None:
ts = time() - 60 * 60 * 24 * 14
end_date_str = from_timestamp(ts)
con = connect_db()
cur = con.cursor()
trf = TfidfVectorizer(
ngram_range=(2,6),
stop_words='english',
analyzer='char',
preprocessor=preprocess_post
)
reg = LogisticRegression()
clf = Pipeline([('vectorizer', trf), ('reg', reg)])
X_raw = []
Y_raw = []
# Fetch closed questions from database
query = """SELECT * FROM questions WHERE creation_date < '{}' AND
closed_reason='off-topic' AND (closed_desc LIKE '%context%'
OR closed_desc LIKE '%homework%');""".format(end_date_str)
cur.execute(query)
for q in cur:
X_raw.append(q['body_html'])
Y_raw.append(1)
num_closed = len(X_raw)
# Fetch an equal number of un-closed questions
query = """SELECT * FROM questions WHERE creation_date < %s AND
closed_reason IS NULL ORDER BY creation_date LIMIT %s"""
cur.execute(query, [end_date_str, num_closed])
for q in cur:
X_raw.append(q['body_html'])
Y_raw.append(0)
X_raw = [X_raw[i] for i in shuff]
Y_raw = [Y_raw[i] for i in shuff]
# Hold back 20% of examples as test set
X_train, X_test, Y_train, Y_test = train_test_split(
X_raw, Y_raw, test_size=0.2)
test_size = len(X_test)
train_size = len(X_train)
# Perform grid search to tune parameters for F1-score
params = [
{
'vectorizer__ngram_range': [(2,2), (2,4), (2,6), (2,8)],
'reg__penalty': ['l1', 'l2'],
'reg__C': [.01, .03, .1, .3, 1, 3, 10, 30, 100],
'reg__intercept_scaling': [.1,1,10,100]
}
]
gridsearch = GridSearchCV(clf, params, scoring='f1', n_jobs=4, \
pre_dispatch=8)
gridsearch.fit(X_train, Y_train)
clf = gridsearch.best_estimator_
print("Done training classifier!")
print("Parameters from CV:")
for k,v in gridsearch.best_params_.items():
print("{}: {}".format(k,v))
preds = clf.predict(X_test)
print("Done making predictions for test set.")
print("Results:")
clf.stats = dict()
clf.stats['train_size'] = train_size
clf.stats['train_pos'] = np.sum(Y_train)
clf.stats['train_neg'] = train_size - np.sum(Y_train)
clf.stats['test_size'] = test_size
clf.stats['test_pos'] = np.sum(Y_test)
clf.stats['test_neg'] = test_size - np.sum(Y_test)
clf.stats['accuracy'] = clf.score(X_test, Y_test)
clf.stats['precision'] = precision_score(Y_test, preds)
clf.stats['recall'] = recall_score(Y_test, preds)
for k in clf.stats:
print(" {}: {}".format(k, clf.stats[k]))
with open('psq.pickle', 'wb') as f:
pickle.dump(clf, f)
def process_api_questions(items, check_quality=True):
"""Add question/answer list to the database.
items -- 'items' object returned from Math.SE query against questions with
specific filters set (as in fetch_recent_questions)
check_quality -- if True, questions are run through the effort predictor
"""
if len(items) == 0:
return
con = connect_db()
cur = con.cursor()
qids = [item['question_id'] for item in items]
qids.sort()
query = "SELECT id FROM questions WHERE id IN ("
query += ','.join([str(i) for i in qids])
query += ');'
cur.execute(query)
existing = [q['id'] for q in cur]
for item in items:
question = {
'id': item['question_id'],
'body_html': item['body'],
'body_markdown': item['body_markdown'],
'creation_date': from_timestamp(item['creation_date']),
'last_activity_date':from_timestamp(item['last_activity_date']),
'link': item['link'],
'score': item['score'],
'title': item['title'],
'historic': True
}
if 'owner' in item and len(item['owner']) > 0:
question['author_id'] = item['owner']['user_id']
else:
question['author_id'] = None
if 'accepted_answer_id' in item:
question['accepted_answer_id'] = item['accepted_answer_id']
else:
question['accepted_answer_id'] = None
if 'closed_date' in item:
question['closed_date'] = from_timestamp(item['closed_date'])
question['closed_desc'] = item['closed_details']['description']
question['closed_reason'] = item['closed_reason']
else:
question['closed_date'] = None
question['closed_desc'] = None
question['closed_reason'] = None
if question['id'] not in existing:
query = 'INSERT INTO questions ('
query += ','.join(sorted(question.keys()))
query += ') VALUES ('
query += ','.join('%s' for i in range(len(question.keys())))
query += ');'
else:
query = "UPDATE questions SET "
query += ','.join(k + '=%s' for k in sorted(question.keys()))
query += ' WHERE id=' + str(question['id']) + ';'
cur.execute(query, [question[k] for k in sorted(question.keys())])
query = "DELETE FROM question_tags WHERE question_id=%s"
cur.execute(query, [question['id']])
query = "SELECT id FROM tags WHERE tags.name IN ("
query += ','.join("'{}'".format(t) for t in item['tags'])
query += ");"
cur.execute(query)
tagids = [t['id'] for t in cur]
query = "INSERT INTO question_tags (question_id, tag_id) VALUES (%s,%s)"
for tagid in tagids:
cur.execute(query, [question['id'], tagid])
if 'answers' in item:
answers = []
for answer in item['answers']:
a = {
'id': answer['answer_id'],
'body_html': answer['body'],
'body_markdown': answer['body_markdown'],
'creation_date':from_timestamp(answer['creation_date']),
'is_accepted': answer['is_accepted'],
'last_activity_date': from_timestamp(answer['last_activity_date']),
'link': answer['link'],
'question_id': answer['question_id'],
'score': answer['score']
}
if 'owner' in answer and len(answer['owner']) > 0:
a['author_id'] = answer['owner']['user_id']
else:
a['author_id'] = None
answers.append(a)
query = "SELECT id FROM answers WHERE id IN ("
query += ','.join(str(a['id']) for a in answers)
query += ') ORDER BY id ASC;'
cur.execute(query)
existinga = [a['id'] for a in cur]
for a in answers:
if a['id'] not in existinga:
query = 'INSERT INTO answers ('
query += ','.join(sorted(a.keys()))
query += ') VALUES ('
query += ','.join('%s' for i in range(len(a.keys())))
query += ');'
else:
query = 'UPDATE answers SET '
query += ','.join(k + '=%s' for k in sorted(a.keys()))
query += ' WHERE id=' + str(a['id']) + ';'
cur.execute(query, [a[k] for k in sorted(a.keys())])
con.commit()
if check_quality:
query = "SELECT id, body_html FROM questions WHERE id IN ("
query += ','.join([str(i) for i in qids])
query += ') ORDER BY id ASC;'
cur.execute(query)
quals = []
bodies = [item['body_html'] for item in cur]
with open('psq.pickle', 'rb') as f:
clf = pickle.load(f)
probs = clf.predict_proba(bodies)[:, 0]
for i in range(len(qids)):
query = "UPDATE questions SET quality_score=%s WHERE id=%s".format(
int(100*probs[i]), qids[i])
cur.execute(query, [int(100*probs[i]), qids[i]])
con.commit()
con.close()
