def newspaper_info(request):
    info = cache.get("newspaper_info")
    if info is None:
        total_page_count = index.page_count()
        titles_with_issues = models.Title.objects.filter(has_issues=True)
        titles_with_issues_count = titles_with_issues.count()

        _places = models.Place.objects.filter(titles__in=titles_with_issues)
        states_with_issues = sorted(set(place.state for place in _places if place.state is not None))

        _languages = models.Language.objects.filter(titles__in=titles_with_issues)
        languages_with_issues = sorted(set((lang.code, lang.name) for lang in _languages))

        # TODO: might make sense to add a Ethnicity.has_issue model field
        # to save having to recompute this all the time, eventhough it 
        # shouldn't take more than 1/2 a second, it all adds up eh?
        ethnicities_with_issues = []
        for e in models.Ethnicity.objects.all():
            # fliter out a few ethnicities, not sure why really
            # https://rdc.lctl.gov/trac/chronam/ticket/724#comment:22
            if e.has_issues and e.name not in ["African", "Canadian", "Welsh"]:
                ethnicities_with_issues.append(e.name)

        info = {'titles_with_issues_count': titles_with_issues_count,
                'states_with_issues': states_with_issues,
                'languages_with_issues': languages_with_issues,
                'ethnicities_with_issues': ethnicities_with_issues,
                'total_page_count': total_page_count}

        cache.set("newspaper_info", info)        
    
    return info
Beispiel #2
0
def status(request):
    page_title = "System Status"
    page_count = models.Page.objects.all().count()
    issue_count = models.Issue.objects.all().count()
    batch_count = models.Batch.objects.all().count()
    title_count = models.Title.objects.all().count()
    holding_count = models.Holding.objects.all().count()
    essay_count = models.Essay.objects.all().count()
    pages_indexed = index.page_count()
    titles_indexed = index.title_count()
    return render_to_response("reports/status.html", dictionary=locals(), context_instance=RequestContext(request))
Beispiel #3
0
def status(request):
    page_title = 'System Status'
    page_count = models.Page.objects.all().count()
    issue_count = models.Issue.objects.all().count()
    batch_count = models.Batch.objects.all().count()
    title_count = models.Title.objects.all().count()
    holding_count = models.Holding.objects.all().count()
    essay_count = models.Essay.objects.all().count()
    pages_indexed = index.page_count()
    titles_indexed = index.title_count()
    return render_to_response('reports/status.html', dictionary=locals(),
                              context_instance=RequestContext(request))
Beispiel #4
0
def status(request):
    crumbs = list(settings.BASE_CRUMBS)
    crumbs.extend([
        {'label': 'System Status'},
    ])
    page_title = 'System Status'
    page_count = models.Page.objects.all().count()
    issue_count = models.Issue.objects.all().count()
    batch_count = models.Batch.objects.all().count()
    title_count = models.Title.objects.all().count()
    # holding_count = models.Holding.objects.all().count()
    # essay_count = models.Essay.objects.all().count()
    pages_indexed = index.page_count()
    titles_indexed = index.title_count()
    return render_to_response('reports/status.html', dictionary=locals(),
                              context_instance=RequestContext(request))
Beispiel #5
0
    def handle(self, **options):
        if not (models.Title.objects.all().count() == 0
                and models.Holding.objects.all().count() == 0
                and models.Essay.objects.all().count() == 0
                and models.Batch.objects.all().count() == 0
                and models.Issue.objects.all().count() == 0
                and models.Page.objects.all().count() == 0
                and index.page_count() == 0 and index.title_count() == 0):
            _logger.warn("Database or index not empty as expected.")
            return

        start = datetime.now()
        management.call_command('loaddata', 'languages.json')
        management.call_command('loaddata', 'institutions.json')
        management.call_command('loaddata', 'ethnicities.json')
        management.call_command('loaddata', 'labor_presses.json')
        management.call_command('loaddata', 'countries.json')

        bib_in_settings = validate_bib_dir()
        if bib_in_settings:
            # look in BIB_STORAGE for original titles to load
            for filename in os.listdir(bib_in_settings):
                if filename.startswith('titles-') and filename.endswith(
                        '.xml'):
                    filepath = os.path.join(bib_in_settings, filename)
                    management.call_command('load_titles',
                                            filepath,
                                            skip_index=True)

        management.call_command(
            'title_sync',
            skip_essays=options['skip_essays'],
            pull_title_updates=options['pull_title_updates'])

        end = datetime.now()
        total_time = end - start
        _logger.info('start time: %s' % start)
        _logger.info('end time: %s' % end)
        _logger.info('total time: %s' % total_time)
        _logger.info("chronam_sync done.")
Beispiel #6
0
    def handle(self, **options):
        if not (models.Title.objects.all().count() == 0 and
                models.Holding.objects.all().count() == 0 and
                models.Essay.objects.all().count() == 0 and
                models.Batch.objects.all().count() == 0 and
                models.Issue.objects.all().count() == 0 and
                models.Page.objects.all().count() == 0 and
                index.page_count() == 0 and
                index.title_count() == 0):
            _logger.warn("Database or index not empty as expected.")
            return

        start = datetime.now()
        management.call_command('loaddata', 'languages.json')
        management.call_command('loaddata', 'institutions.json')
        management.call_command('loaddata', 'ethnicities.json')
        management.call_command('loaddata', 'labor_presses.json')
        management.call_command('loaddata', 'countries.json')

        bib_in_settings = validate_bib_dir()
        if bib_in_settings:
            # look in BIB_STORAGE for original titles to load
            for filename in os.listdir(bib_in_settings):
                if filename.startswith('titles-') and filename.endswith('.xml'):
                    filepath = os.path.join(bib_in_settings, filename)
                    management.call_command('load_titles', filepath, skip_index=True)

        management.call_command('title_sync', 
                                skip_essays=options['skip_essays'],
                                pull_title_updates=options['pull_title_updates'])

        

        end = datetime.now()
        total_time = end - start
        _logger.info('start time: %s' % start)
        _logger.info('end time: %s' % end)
        _logger.info('total time: %s' % total_time)
        _logger.info("chronam_sync done.")
Beispiel #7
0
def newspaper_info(request):
    info = cache.get("newspaper_info")
    if info is None:
        total_page_count = index.page_count()
        titles_with_issues = models.Title.objects.filter(has_issues=True)
        titles_with_issues_count = titles_with_issues.count()

        _places = models.Place.objects.filter(titles__in=titles_with_issues)
        states_with_issues = sorted(
            set(place.state for place in _places if place.state is not None))

        _languages = models.Language.objects.filter(
            titles__in=titles_with_issues)
        languages_with_issues = sorted(
            set((lang.code, lang.name) for lang in _languages))

        # TODO: might make sense to add a Ethnicity.has_issue model field
        # to save having to recompute this all the time, eventhough it
        # shouldn't take more than 1/2 a second, it all adds up eh?
        ethnicities_with_issues = []
        for e in models.Ethnicity.objects.all():
            # fliter out a few ethnicities, not sure why really
            # https://rdc.lctl.gov/trac/chronam/ticket/724#comment:22
            if e.has_issues and e.name not in ["African", "Canadian", "Welsh"]:
                ethnicities_with_issues.append(e.name)

        info = {
            'titles_with_issues_count': titles_with_issues_count,
            'states_with_issues': states_with_issues,
            'languages_with_issues': languages_with_issues,
            'ethnicities_with_issues': ethnicities_with_issues,
            'total_page_count': total_page_count
        }

        cache.set("newspaper_info", info)

    return info
Beispiel #8
0
def newspapers(request, state=None, format='html'):
    if state and state != "all_states":
        state = unpack_url_path(state)
        if state is None:
            raise Http404
        else:
            state = state.title()
    else:
        state = request.REQUEST.get('state', None)

    language = request.REQUEST.get('language', None)
    if language:
        language_display = models.Language.objects.get(code__contains=language).name
    ethnicity = request.REQUEST.get('ethnicity', None)

    if not state and not language and not ethnicity:
        page_title = 'All Digitized Newspapers'
        number_of_pages = index.page_count()
    else:
        page_title = 'Results: Digitized Newspapers'
        
    titles = models.Title.objects.filter(has_issues=True)
    titles = titles.annotate(first=Min('issues__date_issued'))
    titles = titles.annotate(last=Max('issues__date_issued'))

    if state:
        titles = titles.filter(places__state__iexact=state)

    if language:
        titles = titles.filter(languages__code__contains=language)

    if ethnicity:
        try:
            e = models.Ethnicity.objects.get(name=ethnicity)
            ethnicity_filter = Q(subjects__heading__icontains=ethnicity)
            for s in e.synonyms.all():
                ethnicity_filter |= Q(subjects__heading__icontains=s.synonym)
            titles = titles.filter(ethnicity_filter)
        except models.Ethnicity.DoesNotExist:
            pass

    _newspapers_by_state = {}
    for title in titles:
        if state:
            _newspapers_by_state.setdefault(state, set()).add(title)
        else:
            for place in title.places.all():
                if place.state:
                    _newspapers_by_state.setdefault(place.state, set()).add(title)

    newspapers_by_state = [(s, sorted(t, key=lambda title: title.name)) for s, t in sorted(_newspapers_by_state.iteritems())]
    crumbs = list(settings.BASE_CRUMBS)

    if format == "html":
        return render_to_response("newspapers.html",
                                  dictionary=locals(),
                                  context_instance=RequestContext(request))
    elif format == "txt":
        host = request.get_host()
        return render_to_response("newspapers.txt",
                                  dictionary=locals(),
                                  context_instance=RequestContext(request),
                                  mimetype="text/plain")
    elif format == "json":
        host = request.get_host()
        results = {"newspapers": []}
        for state, titles in newspapers_by_state:
            for title in titles:
                results["newspapers"].append({"lccn": title.lccn, "title": title.display_name, "url": "http://" + host + title.json_url, "state": state})

        return HttpResponse(json.dumps(results, indent=2), mimetype='application/json')
    else:
        return HttpResponseServerError("unsupported format: %s" % format)
Beispiel #9
0
def total_page_count(request):
    return JsonResponse({"total_page_count": index.page_count()})