def get(self): id = self.request.get('id') miner = Miner('/comments/' + id) tree = miner.populate() story = Story().getById(id) story.status = 'archived' story.name = tree['data']['title'] story.comments = tree['data']['num_comments'] story.permalink = tree['data']['permalink'] story.compressed = True story.json = base64.b64encode(zlib.compress(json.dumps(tree), 9)) story.put()
def get(self): result = dict() id = self.request.get('id') story = Story().getById(id) if story is False: #Just take a little peek for sanity's sake miner = Miner('/comments/' + id) info = miner.get_info() if info: if info['data']['num_comments'] > 300: taskqueue.add(url='/worker/save_story?id=' + id, method='GET') story = Story() story.id = id story.status = 'queued' story.put() result['message'] = 'Story added to queue for archiving' else: result['message'] = "That story doesn't have over 300 comments" else: result['message'] = "That story doesn't seem to exist. What does existence even mean?" elif story.status is 'queued': result['message'] = 'Still in the queue to be archived' else: result['message'] = 'Story already archived' self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(result))
def get(self): stories = Story.all().filter('status = ', 'archived').order('-comments') self.response.out.write( template.render(TEMPLATE_DIR + 'index.html', { 'stories': stories, 'page': 'index' }))
def get(self): id = self.request.get('id') story = Story().getById(id) result = json.dumps(False) if story: if story.compressed: result = zlib.decompress(base64.b64decode(story.json)) else: result = story.json self.response.headers['Content-Type'] = 'application/json' self.response.out.write(result)
def get_articles_by_story(self, story_id): story=Story.find_by_id(story_id) if story==None: s=relegence_API.stories.by_story_id(story_id, {'numDocs': 100}) story=smodel=Story(story_id=s['id'], title=s['title'], mag_score=s['magScore'], num_total_docs=s['numTotalDocs'], num_original_docs=s['numOriginalDocs'] # ,topic_creation_time= s['topicCreationTime'], last_topic_activity_time = s['lastTopicActivityTime'] ) smodel.save() articles=s['articles'] for a in articles: fname=a['id'].replace('/','|') amodel=Article(article_id=fname, story=smodel, source=a['source']['title'], \ link=a['link'], title=a['title'], snippet=a['snippet'], \ author=a['author']['name'], text=a['content'], file_name=fname, \ source_id=a['source']['id']) amodel.save() return Article.find_by_story(story)
def run_entity_dive_method(articles_collection): ent_store = AnnotationStore('Entity') kt_store = AnnotationStore('KeyTerm') doc_store = DocumentStore('~') prs = [ DuplicateClearingPR(), SentimentAnalyserPR('Sentence'), SentimentHighlighter(), KeyTermAnnotatorPR(), RelEntityTagger(), BratEmbeddingToMongoPR(['KeyTerm', 'PosSentiment', 'NegSentiment', 'Entity']), ent_store, kt_store, doc_store] pipe = Pipeline(articles_collection) pipe.setPRs(prs).setCorpus(articles_collection) result = pipe.process() ent_annots = ent_store.annots # kt_annots= kt_store.annots # unique_kt= set([kt.text for kt in kt_annots]) unique_ent_dict= defaultdict(lambda :[]) for ent in ent_annots: unique_ent_dict[ent['wikidata']].append(ent.text) for key, ent_texts in unique_ent_dict.iteritems(): ename=sorted(ent_texts, key=len)[-1] unique_ent_dict[key]=ename clust_dicts={} for ent_id, ename in unique_ent_dict.iteritems(): fe = CustomEntityFeatureExtractor(ent_id) fann_store=AnnotationStore('Sentence', filterby_attrib_exist='FSentence') Pipeline(result).setPRs([fe, fann_store]).process() fsents=fann_store.annots X = get_sentiment(fsents) clust_dict = cluster_by_sentiment(fsents, X) clust_dicts[ename]=clust_dict clutering_list_model = make_clustering_list_model(clust_dicts) clutering_list_model.collection_id=articles_collection.story_id clutering_list_model.name= Story.find_by_id(articles_collection.story_id).title return clutering_list_model
def get(self): stories = Story.all().filter('status = ', 'archived').order('-comments') self.response.out.write(template.render(TEMPLATE_DIR + 'index.html', {'stories' : stories, 'page': 'index'}))