def newspaper_info(request):
info = cache.get("newspaper_info")
if info is None:
total_page_count = index.page_count()
titles_with_issues = models.Title.objects.filter(has_issues=True)
titles_with_issues_count = titles_with_issues.count()
_places = models.Place.objects.filter(titles__in=titles_with_issues)
states_with_issues = sorted(set(place.state for place in _places if place.state is not None))
_languages = models.Language.objects.filter(titles__in=titles_with_issues)
languages_with_issues = sorted(set((lang.code, lang.name) for lang in _languages))
# TODO: might make sense to add a Ethnicity.has_issue model field
# to save having to recompute this all the time, eventhough it
# shouldn't take more than 1/2 a second, it all adds up eh?
ethnicities_with_issues = []
for e in models.Ethnicity.objects.all():
# fliter out a few ethnicities, not sure why really
# https://rdc.lctl.gov/trac/chronam/ticket/724#comment:22
if e.has_issues and e.name not in ["African", "Canadian", "Welsh"]:
ethnicities_with_issues.append(e.name)
info = {'titles_with_issues_count': titles_with_issues_count,
'states_with_issues': states_with_issues,
'languages_with_issues': languages_with_issues,
'ethnicities_with_issues': ethnicities_with_issues,
'total_page_count': total_page_count}
cache.set("newspaper_info", info)
return info
def status(request):
page_title = "System Status"
page_count = models.Page.objects.all().count()
issue_count = models.Issue.objects.all().count()
batch_count = models.Batch.objects.all().count()
title_count = models.Title.objects.all().count()
holding_count = models.Holding.objects.all().count()
essay_count = models.Essay.objects.all().count()
pages_indexed = index.page_count()
titles_indexed = index.title_count()
return render_to_response("reports/status.html", dictionary=locals(), context_instance=RequestContext(request))
def status(request):
page_title = 'System Status'
page_count = models.Page.objects.all().count()
issue_count = models.Issue.objects.all().count()
batch_count = models.Batch.objects.all().count()
title_count = models.Title.objects.all().count()
holding_count = models.Holding.objects.all().count()
essay_count = models.Essay.objects.all().count()
pages_indexed = index.page_count()
titles_indexed = index.title_count()
return render_to_response('reports/status.html', dictionary=locals(),
context_instance=RequestContext(request))
def status(request):
crumbs = list(settings.BASE_CRUMBS)
crumbs.extend([
{'label': 'System Status'},
])
page_title = 'System Status'
page_count = models.Page.objects.all().count()
issue_count = models.Issue.objects.all().count()
batch_count = models.Batch.objects.all().count()
title_count = models.Title.objects.all().count()
# holding_count = models.Holding.objects.all().count()
# essay_count = models.Essay.objects.all().count()
pages_indexed = index.page_count()
titles_indexed = index.title_count()
return render_to_response('reports/status.html', dictionary=locals(),
context_instance=RequestContext(request))
def handle(self, **options):
if not (models.Title.objects.all().count() == 0
and models.Holding.objects.all().count() == 0
and models.Essay.objects.all().count() == 0
and models.Batch.objects.all().count() == 0
and models.Issue.objects.all().count() == 0
and models.Page.objects.all().count() == 0
and index.page_count() == 0 and index.title_count() == 0):
_logger.warn("Database or index not empty as expected.")
return
start = datetime.now()
management.call_command('loaddata', 'languages.json')
management.call_command('loaddata', 'institutions.json')
management.call_command('loaddata', 'ethnicities.json')
management.call_command('loaddata', 'labor_presses.json')
management.call_command('loaddata', 'countries.json')
bib_in_settings = validate_bib_dir()
if bib_in_settings:
# look in BIB_STORAGE for original titles to load
for filename in os.listdir(bib_in_settings):
if filename.startswith('titles-') and filename.endswith(
'.xml'):
filepath = os.path.join(bib_in_settings, filename)
management.call_command('load_titles',
filepath,
skip_index=True)
management.call_command(
'title_sync',
skip_essays=options['skip_essays'],
pull_title_updates=options['pull_title_updates'])
end = datetime.now()
total_time = end - start
_logger.info('start time: %s' % start)
_logger.info('end time: %s' % end)
_logger.info('total time: %s' % total_time)
_logger.info("chronam_sync done.")
def handle(self, **options):
if not (models.Title.objects.all().count() == 0 and
models.Holding.objects.all().count() == 0 and
models.Essay.objects.all().count() == 0 and
models.Batch.objects.all().count() == 0 and
models.Issue.objects.all().count() == 0 and
models.Page.objects.all().count() == 0 and
index.page_count() == 0 and
index.title_count() == 0):
_logger.warn("Database or index not empty as expected.")
return
start = datetime.now()
management.call_command('loaddata', 'languages.json')
management.call_command('loaddata', 'institutions.json')
management.call_command('loaddata', 'ethnicities.json')
management.call_command('loaddata', 'labor_presses.json')
management.call_command('loaddata', 'countries.json')
bib_in_settings = validate_bib_dir()
if bib_in_settings:
# look in BIB_STORAGE for original titles to load
for filename in os.listdir(bib_in_settings):
if filename.startswith('titles-') and filename.endswith('.xml'):
filepath = os.path.join(bib_in_settings, filename)
management.call_command('load_titles', filepath, skip_index=True)
management.call_command('title_sync',
skip_essays=options['skip_essays'],
pull_title_updates=options['pull_title_updates'])
end = datetime.now()
total_time = end - start
_logger.info('start time: %s' % start)
_logger.info('end time: %s' % end)
_logger.info('total time: %s' % total_time)
_logger.info("chronam_sync done.")
def newspaper_info(request):
info = cache.get("newspaper_info")
if info is None:
total_page_count = index.page_count()
titles_with_issues = models.Title.objects.filter(has_issues=True)
titles_with_issues_count = titles_with_issues.count()
_places = models.Place.objects.filter(titles__in=titles_with_issues)
states_with_issues = sorted(
set(place.state for place in _places if place.state is not None))
_languages = models.Language.objects.filter(
titles__in=titles_with_issues)
languages_with_issues = sorted(
set((lang.code, lang.name) for lang in _languages))
# TODO: might make sense to add a Ethnicity.has_issue model field
# to save having to recompute this all the time, eventhough it
# shouldn't take more than 1/2 a second, it all adds up eh?
ethnicities_with_issues = []
for e in models.Ethnicity.objects.all():
# fliter out a few ethnicities, not sure why really
# https://rdc.lctl.gov/trac/chronam/ticket/724#comment:22
if e.has_issues and e.name not in ["African", "Canadian", "Welsh"]:
ethnicities_with_issues.append(e.name)
info = {
'titles_with_issues_count': titles_with_issues_count,
'states_with_issues': states_with_issues,
'languages_with_issues': languages_with_issues,
'ethnicities_with_issues': ethnicities_with_issues,
'total_page_count': total_page_count
}
cache.set("newspaper_info", info)
return info
def newspapers(request, state=None, format='html'):
if state and state != "all_states":
state = unpack_url_path(state)
if state is None:
raise Http404
else:
state = state.title()
else:
state = request.REQUEST.get('state', None)
language = request.REQUEST.get('language', None)
if language:
language_display = models.Language.objects.get(code__contains=language).name
ethnicity = request.REQUEST.get('ethnicity', None)
if not state and not language and not ethnicity:
page_title = 'All Digitized Newspapers'
number_of_pages = index.page_count()
else:
page_title = 'Results: Digitized Newspapers'
titles = models.Title.objects.filter(has_issues=True)
titles = titles.annotate(first=Min('issues__date_issued'))
titles = titles.annotate(last=Max('issues__date_issued'))
if state:
titles = titles.filter(places__state__iexact=state)
if language:
titles = titles.filter(languages__code__contains=language)
if ethnicity:
try:
e = models.Ethnicity.objects.get(name=ethnicity)
ethnicity_filter = Q(subjects__heading__icontains=ethnicity)
for s in e.synonyms.all():
ethnicity_filter |= Q(subjects__heading__icontains=s.synonym)
titles = titles.filter(ethnicity_filter)
except models.Ethnicity.DoesNotExist:
pass
_newspapers_by_state = {}
for title in titles:
if state:
_newspapers_by_state.setdefault(state, set()).add(title)
else:
for place in title.places.all():
if place.state:
_newspapers_by_state.setdefault(place.state, set()).add(title)
newspapers_by_state = [(s, sorted(t, key=lambda title: title.name)) for s, t in sorted(_newspapers_by_state.iteritems())]
crumbs = list(settings.BASE_CRUMBS)
if format == "html":
return render_to_response("newspapers.html",
dictionary=locals(),
context_instance=RequestContext(request))
elif format == "txt":
host = request.get_host()
return render_to_response("newspapers.txt",
dictionary=locals(),
context_instance=RequestContext(request),
mimetype="text/plain")
elif format == "json":
host = request.get_host()
results = {"newspapers": []}
for state, titles in newspapers_by_state:
for title in titles:
results["newspapers"].append({"lccn": title.lccn, "title": title.display_name, "url": "http://" + host + title.json_url, "state": state})
return HttpResponse(json.dumps(results, indent=2), mimetype='application/json')
else:
return HttpResponseServerError("unsupported format: %s" % format)
def total_page_count(request):
return JsonResponse({"total_page_count": index.page_count()})