def newspaper_info(request): info = cache.get("newspaper_info") if info is None: total_page_count = index.page_count() titles_with_issues = models.Title.objects.filter(has_issues=True) titles_with_issues_count = titles_with_issues.count() _places = models.Place.objects.filter(titles__in=titles_with_issues) states_with_issues = sorted(set(place.state for place in _places if place.state is not None)) _languages = models.Language.objects.filter(titles__in=titles_with_issues) languages_with_issues = sorted(set((lang.code, lang.name) for lang in _languages)) # TODO: might make sense to add a Ethnicity.has_issue model field # to save having to recompute this all the time, eventhough it # shouldn't take more than 1/2 a second, it all adds up eh? ethnicities_with_issues = [] for e in models.Ethnicity.objects.all(): # fliter out a few ethnicities, not sure why really # https://rdc.lctl.gov/trac/chronam/ticket/724#comment:22 if e.has_issues and e.name not in ["African", "Canadian", "Welsh"]: ethnicities_with_issues.append(e.name) info = {'titles_with_issues_count': titles_with_issues_count, 'states_with_issues': states_with_issues, 'languages_with_issues': languages_with_issues, 'ethnicities_with_issues': ethnicities_with_issues, 'total_page_count': total_page_count} cache.set("newspaper_info", info) return info
def status(request): page_title = "System Status" page_count = models.Page.objects.all().count() issue_count = models.Issue.objects.all().count() batch_count = models.Batch.objects.all().count() title_count = models.Title.objects.all().count() holding_count = models.Holding.objects.all().count() essay_count = models.Essay.objects.all().count() pages_indexed = index.page_count() titles_indexed = index.title_count() return render_to_response("reports/status.html", dictionary=locals(), context_instance=RequestContext(request))
def status(request): page_title = 'System Status' page_count = models.Page.objects.all().count() issue_count = models.Issue.objects.all().count() batch_count = models.Batch.objects.all().count() title_count = models.Title.objects.all().count() holding_count = models.Holding.objects.all().count() essay_count = models.Essay.objects.all().count() pages_indexed = index.page_count() titles_indexed = index.title_count() return render_to_response('reports/status.html', dictionary=locals(), context_instance=RequestContext(request))
def status(request): crumbs = list(settings.BASE_CRUMBS) crumbs.extend([ {'label': 'System Status'}, ]) page_title = 'System Status' page_count = models.Page.objects.all().count() issue_count = models.Issue.objects.all().count() batch_count = models.Batch.objects.all().count() title_count = models.Title.objects.all().count() # holding_count = models.Holding.objects.all().count() # essay_count = models.Essay.objects.all().count() pages_indexed = index.page_count() titles_indexed = index.title_count() return render_to_response('reports/status.html', dictionary=locals(), context_instance=RequestContext(request))
def handle(self, **options): if not (models.Title.objects.all().count() == 0 and models.Holding.objects.all().count() == 0 and models.Essay.objects.all().count() == 0 and models.Batch.objects.all().count() == 0 and models.Issue.objects.all().count() == 0 and models.Page.objects.all().count() == 0 and index.page_count() == 0 and index.title_count() == 0): _logger.warn("Database or index not empty as expected.") return start = datetime.now() management.call_command('loaddata', 'languages.json') management.call_command('loaddata', 'institutions.json') management.call_command('loaddata', 'ethnicities.json') management.call_command('loaddata', 'labor_presses.json') management.call_command('loaddata', 'countries.json') bib_in_settings = validate_bib_dir() if bib_in_settings: # look in BIB_STORAGE for original titles to load for filename in os.listdir(bib_in_settings): if filename.startswith('titles-') and filename.endswith( '.xml'): filepath = os.path.join(bib_in_settings, filename) management.call_command('load_titles', filepath, skip_index=True) management.call_command( 'title_sync', skip_essays=options['skip_essays'], pull_title_updates=options['pull_title_updates']) end = datetime.now() total_time = end - start _logger.info('start time: %s' % start) _logger.info('end time: %s' % end) _logger.info('total time: %s' % total_time) _logger.info("chronam_sync done.")
def handle(self, **options): if not (models.Title.objects.all().count() == 0 and models.Holding.objects.all().count() == 0 and models.Essay.objects.all().count() == 0 and models.Batch.objects.all().count() == 0 and models.Issue.objects.all().count() == 0 and models.Page.objects.all().count() == 0 and index.page_count() == 0 and index.title_count() == 0): _logger.warn("Database or index not empty as expected.") return start = datetime.now() management.call_command('loaddata', 'languages.json') management.call_command('loaddata', 'institutions.json') management.call_command('loaddata', 'ethnicities.json') management.call_command('loaddata', 'labor_presses.json') management.call_command('loaddata', 'countries.json') bib_in_settings = validate_bib_dir() if bib_in_settings: # look in BIB_STORAGE for original titles to load for filename in os.listdir(bib_in_settings): if filename.startswith('titles-') and filename.endswith('.xml'): filepath = os.path.join(bib_in_settings, filename) management.call_command('load_titles', filepath, skip_index=True) management.call_command('title_sync', skip_essays=options['skip_essays'], pull_title_updates=options['pull_title_updates']) end = datetime.now() total_time = end - start _logger.info('start time: %s' % start) _logger.info('end time: %s' % end) _logger.info('total time: %s' % total_time) _logger.info("chronam_sync done.")
def newspaper_info(request): info = cache.get("newspaper_info") if info is None: total_page_count = index.page_count() titles_with_issues = models.Title.objects.filter(has_issues=True) titles_with_issues_count = titles_with_issues.count() _places = models.Place.objects.filter(titles__in=titles_with_issues) states_with_issues = sorted( set(place.state for place in _places if place.state is not None)) _languages = models.Language.objects.filter( titles__in=titles_with_issues) languages_with_issues = sorted( set((lang.code, lang.name) for lang in _languages)) # TODO: might make sense to add a Ethnicity.has_issue model field # to save having to recompute this all the time, eventhough it # shouldn't take more than 1/2 a second, it all adds up eh? ethnicities_with_issues = [] for e in models.Ethnicity.objects.all(): # fliter out a few ethnicities, not sure why really # https://rdc.lctl.gov/trac/chronam/ticket/724#comment:22 if e.has_issues and e.name not in ["African", "Canadian", "Welsh"]: ethnicities_with_issues.append(e.name) info = { 'titles_with_issues_count': titles_with_issues_count, 'states_with_issues': states_with_issues, 'languages_with_issues': languages_with_issues, 'ethnicities_with_issues': ethnicities_with_issues, 'total_page_count': total_page_count } cache.set("newspaper_info", info) return info
def newspapers(request, state=None, format='html'): if state and state != "all_states": state = unpack_url_path(state) if state is None: raise Http404 else: state = state.title() else: state = request.REQUEST.get('state', None) language = request.REQUEST.get('language', None) if language: language_display = models.Language.objects.get(code__contains=language).name ethnicity = request.REQUEST.get('ethnicity', None) if not state and not language and not ethnicity: page_title = 'All Digitized Newspapers' number_of_pages = index.page_count() else: page_title = 'Results: Digitized Newspapers' titles = models.Title.objects.filter(has_issues=True) titles = titles.annotate(first=Min('issues__date_issued')) titles = titles.annotate(last=Max('issues__date_issued')) if state: titles = titles.filter(places__state__iexact=state) if language: titles = titles.filter(languages__code__contains=language) if ethnicity: try: e = models.Ethnicity.objects.get(name=ethnicity) ethnicity_filter = Q(subjects__heading__icontains=ethnicity) for s in e.synonyms.all(): ethnicity_filter |= Q(subjects__heading__icontains=s.synonym) titles = titles.filter(ethnicity_filter) except models.Ethnicity.DoesNotExist: pass _newspapers_by_state = {} for title in titles: if state: _newspapers_by_state.setdefault(state, set()).add(title) else: for place in title.places.all(): if place.state: _newspapers_by_state.setdefault(place.state, set()).add(title) newspapers_by_state = [(s, sorted(t, key=lambda title: title.name)) for s, t in sorted(_newspapers_by_state.iteritems())] crumbs = list(settings.BASE_CRUMBS) if format == "html": return render_to_response("newspapers.html", dictionary=locals(), context_instance=RequestContext(request)) elif format == "txt": host = request.get_host() return render_to_response("newspapers.txt", dictionary=locals(), context_instance=RequestContext(request), mimetype="text/plain") elif format == "json": host = request.get_host() results = {"newspapers": []} for state, titles in newspapers_by_state: for title in titles: results["newspapers"].append({"lccn": title.lccn, "title": title.display_name, "url": "http://" + host + title.json_url, "state": state}) return HttpResponse(json.dumps(results, indent=2), mimetype='application/json') else: return HttpResponseServerError("unsupported format: %s" % format)
def total_page_count(request): return JsonResponse({"total_page_count": index.page_count()})