def article(request, url_title="index", printed=False, template_name="wikiprox/page.html"):
"""
"""
alt_title = url_title.replace("_", " ")
try:
page = Page.get(url_title)
page.scrub()
except NotFoundError:
page = None
if not page:
try:
page = Page.get(alt_title)
page.scrub()
except NotFoundError:
page = None
if not page:
# might be an author
author_titles = [author.title for author in Author.authors()]
if url_title in author_titles:
return HttpResponseRedirect(reverse("wikiprox-author", args=[url_title]))
elif alt_title in author_titles:
return HttpResponseRedirect(reverse("wikiprox-author", args=[alt_title]))
raise Http404
if (not page.published) and (not settings.MEDIAWIKI_SHOW_UNPUBLISHED):
template_name = "wikiprox/unpublished.html"
elif printed:
template_name = "wikiprox/article-print.html"
else:
template_name = "wikiprox/article.html"
# DDR objects
# show small number of objects, distributed among topics
TOTAL_OBJECTS = 10
PAGE_OBJECTS = 8
try:
terms_objects = page.ddr_terms_objects(size=TOTAL_OBJECTS)
ddr_error = None
except requests.exceptions.ConnectionError:
terms_objects = []
ddr_error = "ConnectionError"
except requests.exceptions.Timeout:
terms_objects = []
ddr_error = "Timeout"
ddr_objects = ddr.distribute_list(terms_objects, PAGE_OBJECTS)
ddr_objects_width = 280
ddr_img_width = ddr_objects_width / (PAGE_OBJECTS / 2)
return render_to_response(
template_name,
{
"page": page,
"ddr_error": ddr_error,
"ddr_objects": ddr_objects,
"ddr_objects_width": ddr_objects_width,
"ddr_img_width": ddr_img_width,
},
context_instance=RequestContext(request),
)
def create_index():
index = set_hosts_index()
logprint('debug', 'creating new index')
index = Index(settings.DOCSTORE_INDEX)
index.create()
logprint('debug', 'creating mappings')
Author.init()
Page.init()
Source.init()
logprint('debug', 'registering doc types')
index.doc_type(Author)
index.doc_type(Page)
index.doc_type(Source)
logprint('debug', 'DONE')
def related_ddr(request, url_title="index", template_name="wikiprox/related-ddr.html"):
"""List of topic terms and DDR objects relating to page
"""
try:
page = Page.get(url_title)
except NotFoundError:
raise Http404
# show small number of objects, distributed among topics
TOTAL_OBJECTS = 10
try:
terms_objects = page.ddr_terms_objects(size=TOTAL_OBJECTS)
ddr_error = None
except requests.exceptions.ConnectionError:
terms_objects = []
ddr_error = "ConnectionError"
except requests.exceptions.Timeout:
terms_objects = []
ddr_error = "Timeout"
ddr_terms_objects = ddr.distribute_dict(terms_objects, TOTAL_OBJECTS)
# Don't show <ul> list of topics (with links) at top
# unless there are more than one
page_topics = [term for term in terms_objects if term["objects"]]
show_topics_ul = len(page_topics) - 1
return render_to_response(
template_name,
{
"page": page,
"ddr_error": ddr_error,
"ddr_terms_objects": ddr_terms_objects,
"show_topics_ul": show_topics_ul,
"DDR_MEDIA_URL": settings.DDR_MEDIA_URL,
},
context_instance=RequestContext(request),
)
def page_cite(request, url_title, template_name="wikiprox/cite.html"):
try:
page = Page.get(url_title)
except NotFoundError:
raise Http404
if (not page.published) and (not settings.MEDIAWIKI_SHOW_UNPUBLISHED):
raise Http404
citation = Citation(page, request)
return render_to_response(template_name, {"citation": citation}, context_instance=RequestContext(request))
def articles(request, format=None):
"""DOCUMENTATION GOES HERE.
"""
data = [
{
'title': article.title,
'url': reverse('wikiprox-api-page', args=([article.url_title]), request=request),
}
for article in Page.pages()
]
return Response(data)
def article(request, url_title, format=None):
"""DOCUMENTATION GOES HERE.
"""
try:
page = Page.get(url_title)
page.scrub()
except NotFoundError:
return Response(status=status.HTTP_404_NOT_FOUND)
categories = [
reverse('wikiprox-api-category', args=([category]), request=request)
for category in page.categories
]
sources = [
reverse('wikiprox-api-source', args=([source_id]), request=request)
for source_id in page.source_ids
]
topic_term_ids = [
'%s/facet/topics/%s/objects/' % (settings.DDR_API, term['id'])
for term in page.topics()
]
authors = [
reverse('wikiprox-api-author', args=([author_titles]), request=request)
for author_titles in page.authors_data['display']
]
data = {
'url_title': page.url_title,
'url': reverse('wikiprox-api-page', args=([page.url_title]), request=request),
'absolute_url': reverse('wikiprox-page', args=([page.url_title]), request=request),
'published_encyc': page.published_encyc,
'modified': page.modified,
'title_sort': page.title_sort,
'prev_page': reverse('wikiprox-api-page', args=([page.prev_page]), request=request),
'next_page': reverse('wikiprox-api-page', args=([page.next_page]), request=request),
'title': page.title,
'body': page.body,
'categories': categories,
'sources': sources,
'coordinates': page.coordinates,
'authors': authors,
'ddr_topic_terms': topic_term_ids,
}
return Response(data)
def articles(report=False, dryrun=False, force=False):
index = set_hosts_index()
logprint('debug', '------------------------------------------------------------------------')
# authors need to be refreshed
logprint('debug', 'getting mw_authors,articles...')
mw_author_titles = Proxy().authors(cached_ok=False)
mw_articles = Proxy().articles_lastmod()
logprint('debug', 'getting es_articles...')
es_articles = Page.pages()
if force:
logprint('debug', 'forcibly update all articles')
articles_update = [page['title'] for page in es_articles]
articles_delete = []
else:
logprint('debug', 'determining new,delete...')
articles_update,articles_delete = Elasticsearch.articles_to_update(
mw_author_titles, mw_articles, es_articles)
logprint('debug', 'mediawiki articles: %s' % len(mw_articles))
logprint('debug', 'elasticsearch articles: %s' % len(es_articles))
logprint('debug', 'articles to update: %s' % len(articles_update))
logprint('debug', 'articles to delete: %s' % len(articles_delete))
if report:
return
logprint('debug', 'adding articles...')
posted = 0
could_not_post = []
for n,title in enumerate(articles_update):
logprint('debug', '--------------------')
logprint('debug', '%s/%s %s' % (n+1, len(articles_update), title))
logprint('debug', 'getting from mediawiki')
mwpage = Proxy().page(title)
try:
existing_page = Page.get(title)
logprint('debug', 'exists in elasticsearch')
except:
existing_page = None
if (mwpage.published or settings.MEDIAWIKI_SHOW_UNPUBLISHED):
page_sources = [source['encyclopedia_id'] for source in mwpage.sources]
for mwsource in mwpage.sources:
logprint('debug', '- source %s' % mwsource['encyclopedia_id'])
source = Source.from_mw(mwsource, title)
if not dryrun:
source.save()
logprint('debug', 'creating page')
page = Page.from_mw(mwpage, page=existing_page)
if not dryrun:
logprint('debug', 'saving')
page.save()
try:
p = Page.get(title)
except NotFoundError:
logprint('error', 'ERROR: Page(%s) NOT SAVED!' % title)
else:
logprint('debug', 'not publishable: %s' % mwpage)
could_not_post.append(mwpage)
if could_not_post:
logprint('debug', '========================================================================')
logprint('debug', 'Could not post these: %s' % could_not_post)
logprint('debug', 'DONE')
def contents(request, template_name="wikiprox/contents.html"):
return render_to_response(template_name, {"articles": Page.pages()}, context_instance=RequestContext(request))
def categories(request, template_name="wikiprox/categories.html"):
return render_to_response(
template_name, {"articles_by_category": Page.pages_by_category()}, context_instance=RequestContext(request)
)