def dashboard_classifier(): docs = db_articles.find() if 'stats' not in cache: stats = defaultdict(Counter) for doc in docs: if 'machine' in doc.get('analysis', {}): timestamp = time.mktime(doc['time_added'].date().timetuple()) stats[timestamp]['remove'] += len( doc['analysis']['machine']['candidates']['remove']) stats[timestamp]['confirm'] += len( doc['analysis']['machine']['candidates']['confirm']) stats_json = [] for day, day_stats in sorted(stats.items()): day_stats = dict(day_stats) day_stats['day'] = day stats_json.append(day_stats) cache['stats'] = stats_json return render_template('dashboard_classifier.html', stats_json=json.dumps(cache))
def dashboard_article(query_id): query = dashboard_query_index[query_id] docs = list(db_articles.find(query['query'])) for doc in docs: doc['page'] = Page.get(doc['pages'][0]) return render_template('dashboard_query.html', query=query, articles=docs)
def moderation_queue(): articles = list(db_articles.find({'state': 'moderated'}) \ .sort([('time_added', -1)])[:50]) for doc in articles: if 'pages' in doc: doc['page'] = Page.get(doc['pages'][0]) return render_template('moderation_queue.html', articles=articles)
def moderation_queue(): articles = list(db_articles.find({'state': 'moderated'}) \ .sort([('time_added', -1)])[:50]) for doc in articles: if 'pages' in doc: doc['page'] = Page.get(doc['pages'][0]) return render_template('moderation_queue.html', articles=articles)
def dashboard_article(query_id): query = dashboard_query_index[query_id] docs = list(db_articles.find(query['query'])) for doc in docs: doc['page'] = Page.get(doc['pages'][0]) return render_template('dashboard_query.html', query=query, articles=docs)
def dashboard_classifier(): docs = db_articles.find() if 'stats' not in cache: stats = defaultdict(Counter) for doc in docs: if 'machine' in doc.get('analysis', {}): timestamp = time.mktime(doc['time_added'].date().timetuple()) stats[timestamp]['remove'] += len(doc['analysis']['machine']['candidates']['remove']) stats[timestamp]['confirm'] += len(doc['analysis']['machine']['candidates']['confirm']) stats_json = [] for day, day_stats in sorted(stats.items()): day_stats = dict(day_stats) day_stats['day'] = day stats_json.append(day_stats) cache['stats'] = stats_json return render_template('dashboard_classifier.html', stats_json=json.dumps(cache))
from ppsay.db import db_articles from ppsay.article import Article docs = db_articles.find() for doc in docs: article = Article(doc) print article.id num_final_candidates =len([x for x in article.analysis['final']['candidates'] if x['state'] not in ['removed', 'removed_ml']]) num_final_constituencies =len([x for x in article.analysis['final']['constituencies'] if x['state'] not in ['removed', 'removed_ml']]) if num_final_candidates == 0 and num_final_constituencies == 0: article.update_stream()
from ppsay.page import Page from ppsay.db import db_articles from urlparse import urlparse for doc in db_articles.find(): if 'page' not in doc or doc['page'] is None: print "Skipping" continue page_ids = [] for url, final_url in zip(doc['page']['urls'], doc['page']['final_urls']): print url, final_url d = doc['page'] d['url'] = url d['final_url'] = final_url d['source'] = doc['source'] d['domain'] = urlparse(final_url).netloc page = Page.get_url(d['url']) if page is None: print "Creating {}".format(d['url']) page = Page(d) page.save() else: print "Exists" page_ids.append(page._id)
""" Check that every article's pages exist. """ from ppsay.db import db_articles, db_pages for article_doc in db_articles.find(): if 'pages' not in article_doc: print article_doc['_id'], "MISSING PAGES" continue for page_id in article_doc['pages']: page_doc = db_pages.find_one({'_id': page_id}) if page_doc is None: print page_id, "PAGE MISSING"
from ppsay.page import Page from ppsay.db import db_articles from urlparse import urlparse for doc in db_articles.find(): if 'page' not in doc or doc['page'] is None: print "Skipping" continue page_ids = [] for url, final_url in zip(doc['page']['urls'], doc['page']['final_urls']): print url, final_url d = doc['page'] d['url'] = url d['final_url'] = final_url d['source'] = doc['source'] d['domain'] = urlparse(final_url).netloc page = Page.get_url(d['url']) if page is None: print "Creating {}".format(d['url']) page = Page(d) page.save() else: print "Exists" page_ids.append(page._id)
from ppsay.domains import get_domain, new_domain from ppsay.db import db_articles for article in db_articles.find({'state': 'approved'}): article_domain = article.get('domain') if article_domain is not None: domain = get_domain(article_domain) if domain is None: new_domain = new_domain(article_domain) print new_domain
from ppsay.db import db_articles from ppsay.article import Article docs = db_articles.find() for doc in docs: if 'pages' not in doc: print "MISSING PAGES" continue article = Article(doc) print article.id article.update_stream()