def get_articles(self, date): """ Returns list of articles between date 0:00 and date 24:00 """ #use select_related = 2 to fetch all vendor data articles_ = Article.objects(vendor__in=current_user.mongodb_user.subscriptions, date__gte=date.date(), date__lt=date.date() + timedelta(days=1)).select_related(2) #mark articles as read/unread and add id field articles_as_dict = [] for a in articles_: #check in database if article has Read Feedback feedback = ReadArticleFeedback.objects(user_id=self.mongodb_user.id, article=a).first() tmp_article = a._data if feedback is None: tmp_article['read'] = False else: tmp_article['read'] = True tmp_article['id'] = a.id articles_as_dict.append(tmp_article) return articles_as_dict
def test_constructor_with_file_wikicorpus(self): #load tf-idf model tfidf_model = tfidfmodel.TfidfModel.load("/vagrant/data/test_tfidf.model") extractor = TfidfFeatureExtractor("/vagrant/data/test") #load tf-idf corpus tfidf_corpus = MmCorpus('/vagrant/data/test_tfidf_corpus.mm') #load lda corpus #lda_corpus = MmCorpus('/media/sdc1/test_dump/result/test_lda_corpus.mm') #load dictionary id2token = Dictionary.load("/vagrant/data/test_wordids.dict") #load article titles document_titles = DocumentTitles.load("/vagrant/test_articles.txt") #Connect to mongo database connect(self.config_['database']['db-name'], username=self.config_['database']['user'], password=self.config_['database']['passwd'], port=self.config_['database']['port']) #Load articles as test corpus user = User.objects(email=u"*****@*****.**").first() ranked_article_ids = (a.article.id for a in RankedArticle.objects(user_id=user.id).only("article")) all_article_ids = set(a.id for a in Article.objects(id__in=ranked_article_ids).only("id")) read_article_ids = set(a.article.id for a in ReadArticleFeedback.objects(user_id=user.id).only("article")) unread_article_ids = all_article_ids - read_article_ids #sample test articles X, y = get_samples(extractor, read_article_ids, unread_article_ids) s,f = X.shape logger.debug("Training with %d samples, %d features, %d marks" % (s, f, len(y))) #train esa model esa_model = CosineEsaModel(tfidf_corpus, document_titles=document_titles, test_corpus=X, test_corpus_targets=y, num_test_corpus=len(y), num_best_features=15, num_features=len(id2token)) for line in esa_model: print repr(line) esa_model.save('/vagrant/data/test_cesa.model') tmp_esa = CosineEsaModel.load('/vagrant/data/test_cesa.model') print tmp_esa