def receive_feedbacks(self, session, feedbacks): """ Receive feedbacks from user The format of feedback is: { "docs": [[doc_id, feedback_value], ...], "kws": [[keyword_id, feedback_value], ...], "dockws": [[keyword_id, doc_id, feedback_value], ...] } """ print "propagation started..." for doc_fb in feedbacks.get("docs", []): doc_id, fb = doc_fb doc = Document.get(doc_id) self.ppgt.fb_from_doc(doc, fb, session) for kw_fb in feedbacks.get("kws", []): kw_id, fb = kw_fb kw = Keyword.get(kw_id) self.ppgt.fb_from_kw(kw, fb, session) for dockw_fb in feedbacks.get("dockws", []): kw_id, doc_id, fb = dockw_fb doc = Document.get(doc_id) kw = Keyword.get(kw_id) self.ppgt.fb_from_dockw(kw, doc, fb, session) # propagation is done # updates the feedback value self.upd.update(session) print "propagation finished"
def receive_feedbacks(self, session, feedbacks): """ Receive feedbacks from user The format of feedback is: { "docs": [[doc_id, feedback_value], ...], "kws": [[keyword_id, feedback_value], ...], "dockws": [[keyword_id, doc_id, feedback_value], ...] } """ print "propagation started..." for doc_fb in feedbacks.get("docs", []): doc_id, fb = doc_fb doc = Document.get(doc_id) self.ppgt.fb_from_doc(doc, fb, session) for kw_fb in feedbacks.get("kws", []): kw_id, fb = kw_fb kw = Keyword.get(kw_id) self.ppgt.fb_from_kw(kw, fb, session) for dockw_fb in feedbacks.get("dockws", []): kw_id, doc_id, fb = dockw_fb doc = Document.get(doc_id) kw = Keyword.get(kw_id) self.ppgt.fb_from_dockw(kw, doc, fb, session) # propagation is done # updates the feedback value self.upd.update(session) print "propagation finished"
def test_rec_fb_from_dockw(self): """ getter/setting for receiving feedback from in-document keyword """ doc = Document.get(1) doc.rec_fb_from_dockw(Keyword.get("redis"), doc, 1, self.session) doc.rec_fb_from_dockw(Keyword.get("database"), doc, .5, self.session) self.assertEqual(doc.fb_from_kw(self.session), { Keyword.get("redis"): 1, Keyword.get("database"): .5 }) #not the right document self.assertRaises(AssertionError, doc.rec_fb_from_dockw, Document.get(2), Keyword.get("redis"), 1, self.session) #python is not a keyword for document#1, error should be raised self.assertRaises(AssertionError, doc.rec_fb_from_dockw, doc, Keyword.get("python"), 1, self.session) #test the weighted sum weights = [ 0.62981539329519109, 0.45460437826405437, 0.62981539329519109 ] self.assertEqual((weights[0] * 1 + weights[1] * .5) / sum(weights), doc.fb_weighted_sum(self.session))
def test_similarity(self): # for doc doc1 = Document.get(1) doc2 = Document.get(2) doc3 = Document.get(3) self.assertAlmostEqual(0.6300877890447911, doc1.similarity_to(doc2)) self.assertAlmostEqual(doc2.similarity_to(doc1), doc1.similarity_to(doc2)) self.assertAlmostEqual(0.31713642199844894, doc1.similarity_to(doc3)) self.assertRaises(NotImplementedError, doc1.similarity_to, doc3, "not implemented metric") # for kw kw1 = Keyword.get("redis") kw2 = Keyword.get("database") kw3 = Keyword.get("python") self.assertAlmostEqual(0.6698544675330306, kw1.similarity_to(kw2)) self.assertAlmostEqual(kw2.similarity_to(kw1), kw1.similarity_to(kw2)) self.assertAlmostEqual(0.2613424459663648, kw1.similarity_to(kw3)) self.assertRaises(NotImplementedError, kw1.similarity_to, kw3, "not implemented metric")
def setUp(self): #make the fmim kw_filters = [self.my_kw_filter] doc_filters = [self.kw_count_filter, self.has_database_filter] self.r = LinRelRecommender(2, 2, 1.0, 0.1, 1.0, 0.1, #the default configuration kw_filters = None, doc_filters = [self.kw_count_filter, self.has_database_filter], **fmim.__dict__) self.session = get_session() self.session.update_kw_feedback(Keyword.get("redis"), .7) self.session.update_kw_feedback(Keyword.get("database"), .6) self.session.update_doc_feedback(Document.get(1), .7) self.session.update_doc_feedback(Document.get(2), .7) self.session.update_doc_feedback(Document.get(8), .7) filtered_kws = self.r._filter_objs(kw_filters, kws = Keyword.all_kws) filtered_docs = self.r._filter_objs(doc_filters, docs = Document.all_docs) kw2doc_submat, kw_ind_map, kw_ind_map_r = self.r._submatrix_and_indexing(filtered_kws, filtered_docs, fmim.kw2doc_m, fmim.kw_ind, fmim.doc_ind) doc2kw_submat, doc_ind_map, doc_ind_map_r = self.r._submatrix_and_indexing(filtered_docs, filtered_kws, fmim.doc2kw_m, fmim.doc_ind, fmim.kw_ind) self.fmim = FeatureMatrixAndIndexMapping(kw_ind_map, doc_ind_map, kw2doc_submat, doc2kw_submat, kw_ind_map_r, doc_ind_map_r)
def test_doc_fb_threshold_filter_with_prefiltering(self): #change the feedback self.session.update_doc_feedback(Document.get(1), .2) self.session.update_doc_feedback(Document.get(2), .0999999) actual = doc_fb_threshold_filter(0.1, self.session, with_fb=True) expected = Document.get_many([1]) self.assertEqual(expected, actual)
def test_doc_fb_threshold_filter(self): #change the feedback self.session.update_doc_feedback(Document.get(1), .2) self.session.update_doc_feedback(Document.get(2), .0999999) actual = doc_fb_threshold_filter(0.1, self.session, docs = Document.all_docs, with_fb = False) expected = Document.get_many([1]) self.assertEqual(expected, actual)
def test_affected_docs(self): docs = [Document.get(1), Document.get(2)] self.session.add_affected_docs(*docs) self.assertEqual(docs, self.session.affected_docs) doc3 = Document.get(3) docs.append(doc3) self.session.add_affected_docs(doc3) self.assertEqual(set(docs), set(self.session.affected_docs))
def test_fb_weighted_sum_dockw_only(self): """ test if the weighted sum is correct only feedback from dockw/doc """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) self.assertEqual((1 + .5) / 2, kw.fb_weighted_sum(self.session))
def setUp(self): self.r = LinRelRecommender(2, 2, 1., .1, 1., .1, None, None, **fmim.__dict__) self.session = get_session() #giving the feedbacks self.session.update_kw_feedback(Keyword.get("redis"), .7) self.session.update_kw_feedback(Keyword.get("database"), .6) self.session.update_doc_feedback(Document.get(1), .7) self.session.update_doc_feedback(Document.get(2), .7) self.session.update_doc_feedback(Document.get(8), .7)
def test_document_centroid(self): doc = Document.get(1) doclist1 = DocumentList([doc]) self.assertArrayAlmostEqual(matrix2array(doclist1.centroid), doc.vec.toarray()[0]) doc1 = Document.get(1) doc2 = Document.get(2) doclist2 = Document.get_many([1, 2]) self.assertArrayAlmostEqual(matrix2array(doclist2.centroid), (doc1.vec.toarray()[0] + doc2.vec.toarray()[0]) / 2)
def test_fb_weighted_sum_dockw_only(self): """ test if the weighted sum is correct only feedback from dockw/doc """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) self.assertEqual((1 + .5) / 2, kw.fb_weighted_sum(self.session))
def test_rec_fb_from_dockw(self): """ getter/setting for receiving feedback from in-document keyword """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(2), .5, self.session) kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) self.assertEqual(kw.fb_from_doc(self.session), {Document.get(1): 1, Document.get(2): .5}) #is not the right keyword self.assertRaises(AssertionError, kw.rec_fb_from_dockw, Keyword.get("the"), Document.get(1), 1, self.session)
def test_rec_fb_from_doc(self): """ getter/setting for receiving feedback from document """ kw = Keyword.get("redis") kw.rec_fb_from_doc(Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) self.assertEqual(kw.fb_from_doc(self.session), {Document.get(1): 1, Document.get(2): .5}) #does not contain redis, error should be raised self.assertRaises(AssertionError, kw.rec_fb_from_doc, Document.get(3), 1, self.session)
def test_fb_weighted_sum_mixed_source(self): """ test if the weighted sum is correct feedback include all three sources """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) kw.rec_fb_from_kw(kw, .5, self.session) self.assertEqual(.3 * (1 / 2. + 1 / 4.) + .7 * .5, kw.fb_weighted_sum(self.session))
def test_document_centroid(self): doc = Document.get(1) doclist1 = DocumentList([doc]) self.assertArrayAlmostEqual(matrix2array(doclist1.centroid), doc.vec.toarray()[0]) doc1 = Document.get(1) doc2 = Document.get(2) doclist2 = Document.get_many([1, 2]) self.assertArrayAlmostEqual( matrix2array(doclist2.centroid), (doc1.vec.toarray()[0] + doc2.vec.toarray()[0]) / 2)
def test_fb_weighted_sum_mixed_source(self): """ test if the weighted sum is correct feedback include all three sources """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) kw.rec_fb_from_kw(kw, .5, self.session) self.assertEqual(.3 * (1 / 2. + 1 / 4.) + .7 * .5, kw.fb_weighted_sum(self.session))
def test_loop_done(self): """ test if things are cleaned when the loop is done """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) kw.rec_fb_from_kw(kw, .5, self.session) # terminate the loop # everything feedback stuff cleaned kw.loop_done(self.session) self.assertEqual(kw.fb_weighted_sum(self.session), 0)
def test_loop_done(self): """ test if things are cleaned when the loop is done """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) kw.rec_fb_from_kw(kw, .5, self.session) # terminate the loop # everything feedback stuff cleaned kw.loop_done(self.session) self.assertEqual(kw.fb_weighted_sum(self.session), 0)
def test_update_doc_fb(self): """update document feedback""" doc = Document.get(1) self.session.update_doc_feedback(doc, 1) self.assertEqual(self.session.doc_feedbacks, {doc: 1}) self.assertEqual(doc.fb(self.session), 1)
def setUp(self): self.r = LinRelRecommender(2, 2, 1., .1, 1., .1, None, None, **fmim.__dict__) self.session = get_session() #giving the feedbacks self.session.update_kw_feedback(Keyword.get("redis"), .7) self.session.update_kw_feedback(Keyword.get("database"), .6) self.session.update_doc_feedback(Document.get(1), .7) self.session.update_doc_feedback(Document.get(2), .7) self.session.update_doc_feedback(Document.get(8), .7)
def test_rec_fb_from_doc(self): """ getter/setting for receiving feedback from document """ kw = Keyword.get("redis") kw.rec_fb_from_doc(Document.get(1), 1, self.session) kw.rec_fb_from_doc(Document.get(2), .5, self.session) self.assertEqual(kw.fb_from_doc(self.session), { Document.get(1): 1, Document.get(2): .5 }) #does not contain redis, error should be raised self.assertRaises(AssertionError, kw.rec_fb_from_doc, Document.get(3), 1, self.session)
def test_rec_from_doc(self): """ getter/setting for receiving feedback from document """ doc = Document.get(1) doc.rec_fb_from_doc(doc, 1, self.session) self.assertEqual(1, doc.fb_from_doc(self.session)) doc.rec_fb_from_doc(doc, .5, self.session) self.assertEqual(.5, doc.fb_from_doc(self.session)) #is not the right document self.assertRaises(AssertionError, doc.rec_fb_from_doc, Document.get(2), 1, self.session) #test the weighted sum self.assertEqual(.5 * .7, doc.fb_weighted_sum(self.session))
def test_rec_fb_from_dockw(self): """ getter/setting for receiving feedback from in-document keyword """ kw = Keyword.get("redis") kw.rec_fb_from_dockw(kw, Document.get(2), .5, self.session) kw.rec_fb_from_dockw(kw, Document.get(1), 1, self.session) self.assertEqual(kw.fb_from_doc(self.session), { Document.get(1): 1, Document.get(2): .5 }) #is not the right keyword self.assertRaises(AssertionError, kw.rec_fb_from_dockw, Keyword.get("the"), Document.get(1), 1, self.session)
def test_fb_from_dockw(self): kw = Keyword.get("redis") doc = Document.get(1) ppgt.fb_from_dockw(kw, doc, .5, self.session) upd.update(self.session) self.assertAlmostEqual(0.183701573217, doc.fb(self.session)) self.assertAlmostEqual(1/4., kw.fb(self.session))
def test_rec_from_doc(self): """ getter/setting for receiving feedback from document """ doc = Document.get(1) doc.rec_fb_from_doc(doc, 1, self.session) self.assertEqual(1, doc.fb_from_doc(self.session)) doc.rec_fb_from_doc(doc, .5, self.session) self.assertEqual(.5, doc.fb_from_doc(self.session)) #is not the right document self.assertRaises(AssertionError, doc.rec_fb_from_doc, Document.get(2), 1, self.session) #test the weighted sum self.assertEqual(.5 * .7, doc.fb_weighted_sum(self.session))
def test_kw_fb_filter(self): kw = Keyword.get("redis") kw.rec_fb_from_doc(Document.get(1), 1, self.session) self.session.add_doc_recom_list(Document.get_many([1, 2, 6])) self.session.update_kw_feedback(kw, kw.fb_weighted_sum(self.session)) actual = FilterRepository.filters["kw_fb"]([kw]) expected = Keyword.get_many(["redis"]) self.assertEqual(expected, actual)
def test_kw_fb_filter(self): kw = Keyword.get("redis") kw.rec_fb_from_doc(Document.get(1), 1, self.session) self.session.add_doc_recom_list(Document.get_many([1, 2, 6])) self.session.update_kw_feedback(kw, kw.fb_weighted_sum(self.session)) actual = FilterRepository.filters["kw_fb"]([kw]) expected = Keyword.get_many(["redis"]) self.assertEqual(expected, actual)
def test_doc_fb_filter(self): doc = Document.get(1) doc.rec_fb_from_kw(Keyword.get("redis"), 1, self.session) self.session.update_doc_feedback(doc, doc.fb_weighted_sum(self.session)) print "doc.fb(self.session)=", doc.fb(self.session) actual = FilterRepository.filters["doc_fb"]([doc]) expected = Document.get_many([]) print doc.fb(self.session) self.assertEqual(expected, actual)
def test_rec_fb_from_dockw(self): """ getter/setting for receiving feedback from in-document keyword """ doc = Document.get(1) doc.rec_fb_from_dockw(Keyword.get("redis"), doc, 1, self.session) doc.rec_fb_from_dockw(Keyword.get("database"), doc, .5, self.session) self.assertEqual(doc.fb_from_kw(self.session), {Keyword.get("redis"): 1, Keyword.get("database"): .5}) #not the right document self.assertRaises(AssertionError, doc.rec_fb_from_dockw, Document.get(2), Keyword.get("redis"), 1, self.session) #python is not a keyword for document#1, error should be raised self.assertRaises(AssertionError, doc.rec_fb_from_dockw, doc, Keyword.get("python"), 1, self.session) #test the weighted sum weights = [0.62981539329519109, 0.45460437826405437, 0.62981539329519109] self.assertEqual((weights[0] * 1 + weights[1] * .5) / sum(weights), doc.fb_weighted_sum(self.session))
def test_model2modellist_similarity(self): #for keywords kw = Keyword.get("redis") kwlist = Keyword.get_many(["database", "mysql"]) self.assertAlmostEqual(0.3754029265429976, kw.similarity_to(kwlist)) #for documents doc = Document.get(6) doclist = Document.get_many([1, 2]) self.assertAlmostEqual(0.7382455893131392, doc.similarity_to(doclist))
def test_model2modellist_similarity(self): #for keywords kw = Keyword.get("redis") kwlist = Keyword.get_many(["database", "mysql"]) self.assertAlmostEqual(0.3754029265429976, kw.similarity_to(kwlist)) #for documents doc = Document.get(6) doclist = Document.get_many([1, 2]) self.assertAlmostEqual(0.7382455893131392, doc.similarity_to(doclist))
def test_fb_from_doc(self): doc = Document.get(1) ppgt.fb_from_doc(doc, 0.5, self.session) upd.update(self.session) # assertions self.assertAlmostEqual(.5 * .7, doc.fb(self.session)) self.assertAlmostEqual(1/2., Keyword.get("a").fb(self.session)) self.assertAlmostEqual(1/4., Keyword.get("redis").fb(self.session)) self.assertAlmostEqual(1/4., Keyword.get("database").fb(self.session))
def test_doc_fb_filter(self): doc = Document.get(1) doc.rec_fb_from_kw(Keyword.get("redis"), 1, self.session) self.session.update_doc_feedback(doc, doc.fb_weighted_sum(self.session)) print "doc.fb(self.session)=", doc.fb(self.session) actual = FilterRepository.filters["doc_fb"]([doc]) expected = Document.get_many([]) print doc.fb(self.session) self.assertEqual(expected, actual)
def setUp(self): #make the fmim kw_filters = [self.my_kw_filter] doc_filters = [self.kw_count_filter, self.has_database_filter] self.r = LinRelRecommender( 2, 2, 1.0, 0.1, 1.0, 0.1, #the default configuration kw_filters=None, doc_filters=[self.kw_count_filter, self.has_database_filter], **fmim.__dict__) self.session = get_session() self.session.update_kw_feedback(Keyword.get("redis"), .7) self.session.update_kw_feedback(Keyword.get("database"), .6) self.session.update_doc_feedback(Document.get(1), .7) self.session.update_doc_feedback(Document.get(2), .7) self.session.update_doc_feedback(Document.get(8), .7) filtered_kws = self.r._filter_objs(kw_filters, kws=Keyword.all_kws) filtered_docs = self.r._filter_objs(doc_filters, docs=Document.all_docs) kw2doc_submat, kw_ind_map, kw_ind_map_r = self.r._submatrix_and_indexing( filtered_kws, filtered_docs, fmim.kw2doc_m, fmim.kw_ind, fmim.doc_ind) doc2kw_submat, doc_ind_map, doc_ind_map_r = self.r._submatrix_and_indexing( filtered_docs, filtered_kws, fmim.doc2kw_m, fmim.doc_ind, fmim.kw_ind) self.fmim = FeatureMatrixAndIndexMapping(kw_ind_map, doc_ind_map, kw2doc_submat, doc2kw_submat, kw_ind_map_r, doc_ind_map_r)
def test_query_that_produces_match(self): """ Query that matches something in the corpus """ query = "python, redis" docs, kws = self.r.recommend(query) self.assertEqual(Document.get(6), docs[0]) self.assertEqual(4, len(docs)) #kws should be superset of assoc_kws assoc_kws = set([kw for doc in docs for kw in doc.keywords]) self.assertTrue(assoc_kws.issubset(set(kws)))
def test_all_together(self): """ All three types of feedbacks are involved """ doc = Document.get(1) kw = Keyword.get("redis") recom_docs = [Document.get(_id) for _id in [1,2,3]] self.session.add_doc_recom_list(recom_docs) ppgt.fb_from_doc(doc, 0.5, self.session) ppgt.fb_from_dockw(kw, doc, .5, self.session) ppgt.fb_from_kw(kw, 0.5, self.session) upd.update(self.session) self.assertAlmostEqual(0.56689342264886755 * .5 / (0.56689342264886755 + 0.49704058656839417), Keyword.get("a").fb(self.session)) self.assertAlmostEqual(1 / 4. * .3 + .5 * .7, Keyword.get("redis").fb(self.session)) self.assertAlmostEqual(1 / 4., Keyword.get("database").fb(self.session)) self.assertAlmostEqual(0.183701573217 * .3 + .7 * .5, recom_docs[0].fb(self.session)) self.assertAlmostEqual(0.191506501383, recom_docs[1].fb(self.session)) self.assertAlmostEqual(0, recom_docs[2].fb(self.session))
def test_fb_from_kw(self): kw = Keyword.get("redis") recom_docs = [Document.get(_id) for _id in [1,2,3]] self.session.add_doc_recom_list(recom_docs) ppgt.fb_from_kw(kw, 0.5, self.session) upd.update(self.session) # assertions self.assertAlmostEqual(.5 * .7, kw.fb(self.session)) self.assertAlmostEqual(0.183701573217, recom_docs[0].fb(self.session)) self.assertAlmostEqual(0.191506501383, recom_docs[1].fb(self.session)) self.assertAlmostEqual(0, recom_docs[2].fb(self.session))
def test_query_that_produces_match(self): """ Query that matches something in the corpus """ query = "python, redis" docs, kws = self.r.recommend(query) self.assertEqual(Document.get(6), docs[0]) self.assertEqual(4, len(docs)) #kws should be superset of assoc_kws assoc_kws = set([kw for doc in docs for kw in doc.keywords]) self.assertTrue(assoc_kws.issubset(set(kws)))
def test_similarity(self): # for doc doc1 = Document.get(1) doc2 = Document.get(2) doc3 = Document.get(3) self.assertAlmostEqual(0.6300877890447911, doc1.similarity_to(doc2)) self.assertAlmostEqual(doc2.similarity_to(doc1), doc1.similarity_to(doc2)) self.assertAlmostEqual(0.31713642199844894, doc1.similarity_to(doc3)) self.assertRaises(NotImplementedError, doc1.similarity_to, doc3, "not implemented metric") # for kw kw1 = Keyword.get("redis") kw2 = Keyword.get("database") kw3 = Keyword.get("python") self.assertAlmostEqual(0.6698544675330306, kw1.similarity_to(kw2)) self.assertAlmostEqual(kw2.similarity_to(kw1), kw1.similarity_to(kw2)) self.assertAlmostEqual(0.2613424459663648, kw1.similarity_to(kw3)) self.assertRaises(NotImplementedError, kw1.similarity_to, kw3, "not implemented metric")
def test_document(self): """ whether id, article, keywords are correct """ kw_strs = ["redis", "database", "a"] doc = Document.get(1) self.assertEqual(doc.id, 1) self.assertEqual(doc.title, "redis: key-value-storage database (ONE)") self.assertEqual(set(doc.keywords), set(Keyword.get_many(kw_strs))) #that is as far as we can test #no numerical testing self.assertTrue(type(doc._kw_weight) is DictType)
def test_loop_done(self): """ test if things are cleaned when the loop is done """ doc = Document.get(1) doc.rec_fb_from_dockw(Keyword.get("redis"), doc, 1, self.session) doc.rec_fb_from_kw(Keyword.get("database"), .5, self.session) doc.rec_fb_from_doc(doc, .5, self.session) # terminate the loop # everything feedback stuff cleaned doc.loop_done(self.session) self.assertEqual(doc.fb_weighted_sum(self.session), 0)
def test_document(self): """ whether id, article, keywords are correct """ kw_strs = ["redis", "database", "a"] doc = Document.get(1) self.assertEqual(doc.id, 1) self.assertEqual(doc.title, "redis: key-value-storage database (ONE)") self.assertEqual(set(doc.keywords), set(Keyword.get_many(kw_strs))) #that is as far as we can test #no numerical testing self.assertTrue(type(doc._kw_weight) is DictType)
def test_loop_done(self): """ test if things are cleaned when the loop is done """ doc = Document.get(1) doc.rec_fb_from_dockw(Keyword.get("redis"), doc, 1, self.session) doc.rec_fb_from_kw(Keyword.get("database"), .5, self.session) doc.rec_fb_from_doc(doc, .5, self.session) # terminate the loop # everything feedback stuff cleaned doc.loop_done(self.session) self.assertEqual(doc.fb_weighted_sum(self.session), 0)
def test_all_together(self): """ All three sources of feedbacks are involved """ doc = Document.get(1) doc.rec_fb_from_dockw(Keyword.get("redis"), doc, 1, self.session) doc.rec_fb_from_kw(Keyword.get("database"), .5, self.session) doc.rec_fb_from_doc(doc, .5, self.session) redis = Keyword.get("redis") db = Keyword.get("database") weights = {redis: 0.62981539329519109, db: 0.45460437826405437, Keyword.get("a"): 0.62981539329519109 } self.assertAlmostEqual(.5 * .7 + .3 * (weights[redis] * 1 + weights[db] * .5) / sum(weights.values()), doc.fb_weighted_sum(self.session))
def recommend_documents(self, fmim, session, top_n, mu, c, sampler = None): """ return a list of document ids as well as the scores """ docs = Document.get_many(fmim.doc_ind.keys()) fbs = dict([(doc.id, doc.fb(session)) for doc in docs]) id_with_scores, id_with_explt_scores, id_with_explr_scores = self.generic_rank(fmim.doc2kw_m, fbs, fmim.doc_ind,fmim.doc_ind_r, mu, c) docs = [] for doc_id, score in id_with_scores.items()[:top_n]: doc = Document.get(doc_id) doc["score"] = score doc['recommended'] = True docs.append(doc) return docs
def recommend_documents(self, query, top_n): """ Param: query: string, the query string, phrases separated by comma, for example: machine learning, natural language processing top_n: integer, the number of documents to be returned Return: DocumentList, the recommended documents KeywordList, the query keywords(that exist in the corpus) """ query_keywords = [kw_str.strip() for kw_str in query.strip().split(",")] #prepare the query word binary column vector word_vec = self._word_vec(query_keywords) existing_keywords = Keyword.get_many([word for word in query_keywords if self.kw_ind.has_key(word)]) #get the scores for documents and score it scores = matrix2array((self.doc2kw_m * word_vec).T) #get none zero scores non_zero_scores = filter(None, scores) sorted_scores = sorted(enumerate(non_zero_scores), key = lambda (_, score): score, reverse = True) #get the top_n documents docs = DocumentList([]) for ind, score in sorted_scores[:top_n]: doc_id = self.doc_ind_r[ind] doc = Document.get(doc_id) doc['score'] = score doc["recommended"] = True docs.append(doc) return docs, existing_keywords
def test_all_together(self): """ All three sources of feedbacks are involved """ doc = Document.get(1) doc.rec_fb_from_dockw(Keyword.get("redis"), doc, 1, self.session) doc.rec_fb_from_kw(Keyword.get("database"), .5, self.session) doc.rec_fb_from_doc(doc, .5, self.session) redis = Keyword.get("redis") db = Keyword.get("database") weights = { redis: 0.62981539329519109, db: 0.45460437826405437, Keyword.get("a"): 0.62981539329519109 } self.assertAlmostEqual( .5 * .7 + .3 * (weights[redis] * 1 + weights[db] * .5) / sum(weights.values()), doc.fb_weighted_sum(self.session))
def test_type_mismatch(self): kw = Keyword.get("redis") kwlist = Keyword.get_many(["database", "mysql"]) doc = Document.get(6) doclist = Document.get_many([1, 2]) #doc to kw self.assertRaises(AssertionError, kw.similarity_to, doc) #kw to doc self.assertRaises(AssertionError, doc.similarity_to, kw) #kw to doclist self.assertRaises(AssertionError, kw.similarity_to, doclist) #doclist to kw self.assertRaises(AssertionError, doclist.similarity_to, kw) #doc to kwlist self.assertRaises(AssertionError, doc.similarity_to, kwlist) #kwlist to doc self.assertRaises(AssertionError, kwlist.similarity_to, doc)
def test_type_mismatch(self): kw = Keyword.get("redis") kwlist = Keyword.get_many(["database", "mysql"]) doc = Document.get(6) doclist = Document.get_many([1, 2]) #doc to kw self.assertRaises(AssertionError, kw.similarity_to, doc) #kw to doc self.assertRaises(AssertionError, doc.similarity_to, kw) #kw to doclist self.assertRaises(AssertionError, kw.similarity_to, doclist) #doclist to kw self.assertRaises(AssertionError, doclist.similarity_to, kw) #doc to kwlist self.assertRaises(AssertionError, doc.similarity_to, kwlist) #kwlist to doc self.assertRaises(AssertionError, kwlist.similarity_to, doc)
from util import config_doc_kw_model config_doc_kw_model() from scinet3.model import Document, Keyword Document.load_all_from_db() def doc_compute(d): return d.get(Keyword.get("redis"), 0) * .5 / sum(d.values()) doc = Document.get(1) print doc._kw_weight {"a": 0.62981539329519109, "redis": 0.62981539329519109, "database": 0.45460437826405437} print doc_compute(doc._kw_weight) doc = Document.get(2) print doc._kw_weight {"the": 0.58478244910295341, "redis": 0.6577450118852588, "database": 0.47476413782131072} print doc_compute(doc._kw_weight) doc = Document.get(3) print doc._kw_weight {"tornado": 0.57555052264377027, "web": 0.50353869621889158, "a": 0.50353869621889158, "python": 0.40204372735417787} print doc_compute(doc._kw_weight) def kw_compute(d):
def doc_feedbacks(self): """document feedback""" key = "session:%s:%s" %(self.session_id, "doc_feedbacks") return dict([(Document.get(int(_id)), float(fb)) for _id, fb in self.redis.hgetall(key).items()])
def affected_docs(self): return [Document.get(doc_id) for doc_id in self.get("affected_docs", set())]