import logging from django.core.cache import cache from django.core.management.base import BaseCommand, CommandError from chronam.core.management.commands import configure_logging configure_logging('', 'purge_django_cache.log' ) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): help = "Purge the django cache after ingest/purge of a batch" def handle(self, *args, **options): try: # delete the total pages count LOGGER.info('removing newspaper_info from cache') cache.delete('newspaper_info') # delete the advanced search title list LOGGER.info('removing titles_states from cache') cache.delete('titles_states') # Delete the fulltext range so the search form is up-to-date LOGGER.info('Removing fulltext_range from cache') cache.delete('fulltext_range') except Exception, e:
import os import logging from optparse import make_option from django.conf import settings from django.db import connection from django.core.management.base import BaseCommand, CommandError from solr import SolrConnection from chronam.core.batch_loader import BatchLoader, BatchLoaderException from chronam.core.management.commands import configure_logging configure_logging('purge_batches_logging.config', 'purge_batch_%s.log' % os.getpid()) log = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--no-optimize', action='store_false', dest='optimize', default=True, help='Do not optimize Solr and MySQL after purge'), ) help = "Purge a batch" args = '<batch_location>' def handle(self, batch_location=None, *args, **options):
import logging import urllib2 from django.core.management.base import BaseCommand from django.db import reset_queries from rdflib import Namespace, ConjunctiveGraph, URIRef try: import simplejson as json except ImportError: import json from chronam.core import models from chronam.core.management.commands import configure_logging configure_logging("chronam_link_places.config", "chronam_link_places.log") _logger = logging.getLogger(__name__) geo = Namespace('http://www.w3.org/2003/01/geo/wgs84_pos#') owl = Namespace('http://www.w3.org/2002/07/owl#') dbpedia = Namespace('http://dbpedia.org/ontology/') class Command(BaseCommand): def handle(self, **options): _logger.debug("linking places") for place in models.Place.objects.filter(dbpedia__isnull=True): if not place.city or not place.state: continue # formulate a dbpedia place uri
import os import logging from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.models import Place, Region configure_logging('refine_places_logging.config', 'refine_places_%s.log' % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + () help = "Parse place names and update city, county and state values. Also add region if possible." def handle(self, *args, **options): metro_atlanta_counties = [ 'Clayton', 'Cobb', 'DeKalb', 'Fulton', 'Gwinnett', 'Henry', 'Rockdale' ] north_georgia_counties = [ 'Banks', 'Barrow', 'Bartow', 'Catoosa', 'Chattooga', 'Cherokee', 'Clarke', 'Dade', 'Dawson', 'Elbert', 'Fannin', 'Floyd', 'Forsyth', 'Franklin', 'Gilmer', 'Gordon', 'Habersham', 'Hall', 'Hart', 'Jackson', 'Lumpkin', 'Madison', 'Morgan', 'Murray', 'Newton', 'Oconee', 'Oglethorpe', 'Pickens', 'Polk', 'Rabun', 'Stephens', 'Towns', 'Union', 'Walker', 'Walton', 'White', 'Whitfield'
import os import logging from datetime import datetime from optparse import make_option from django.core import management from django.core.management.base import BaseCommand from chronam.core import models from chronam.core import index from chronam.core.management.commands import configure_logging from chronam.core.utils.utils import validate_bib_dir configure_logging("chronam_sync_logging.config", "chronam_sync.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): verbose = make_option('--verbose', action='store_true', dest='verbose', default=False, help='') skip_essays = make_option('--skip-essays', action='store_true', dest='skip_essays', default=False, help='Skip essay loading.') pull_title_updates = make_option('--pull-title-updates', action='store_true',
import logging from django.core import management from django.core.management.base import BaseCommand from chronam.core import models from chronam.core.holding_loader import HoldingLoader from chronam.core.management.commands import configure_logging configure_logging('load_holdings_logging.config', 'load_holdings.log') _logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Load a holdings records after title records are all loaded" args = '<location of holdings directory>' def handle(self, holdings_source, *args, **options): # First we want to make sure that our material types are up to date material_types = models.MaterialType.objects.all() [m.delete() for m in material_types] management.call_command('loaddata', 'material_types.json') holding_loader = HoldingLoader() holding_loader.main(holdings_source)
import os import logging from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.management.commands import configure_logging from chronam.core import tasks configure_logging('poll_cts_logging.config', 'poll_cts_%s.log' % os.getpid()) log = logging.getLogger(__name__) class Command(BaseCommand): help = "manual command to load new batches from cts" def handle(self, *args, **options): try: tasks.poll_cts.apply() except Exception, e: log.exception(e) raise CommandError("unable to load batches from cts")
import os import csv import codecs from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.models import Institution configure_logging("load_intitutions_logging.config", "load_institutions.log") """ Loads the institutions based on a CSV file in the form of: <code>, <name>, <city>, <state>, <zip> """ class Command(BaseCommand): help = 'loads institution csv data into Institution table' args = '<institution_csv_file>' def handle(self, csv_file, *args, **options): for row in unicode_csv_reader(codecs.open(csv_file, encoding='utf-8')): i = Institution() i.code = row[0].upper() i.name = row[1] i.address1 = "" i.address2 = "" i.city = row[2] i.state = row[3] i.zip = row[4] i.save()
import os import logging from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.management.commands import configure_logging from chronam.core import tasks configure_logging('poll_purge_logging.config', 'poll_purge_%s.log' % os.getpid()) log = logging.getLogger(__name__) class Command(BaseCommand): help = "manual command to process purge_batch requests from CTS" def handle(self, *args, **options): try: tasks.poll_purge.apply() except Exception, e: log.exception(e) raise CommandError("Unable to purge batches")
import logging from datetime import datetime from optparse import make_option from django.core.management.base import BaseCommand from django.conf import settings from chronam.core import title_pull from chronam.core.management.commands import configure_logging configure_logging('pull_titles_logging.config', 'pull_titles.log') _logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Retrieve a fresh pull of titles from OCLC. \ #TODO: add a list of example commands." args = '' #TODO: Remove default from lccn option_list = BaseCommand.option_list + ( make_option('-l', '--lccn', action='store', dest='lccn', default=None, help="Pass a specific lccn to pull down updates from Worldcat."), make_option('-o', '--oclc', action='store', dest='oclc', default=None,
import os import logging from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core import batch_loader from chronam.core.management.commands import configure_logging configure_logging('process_coordinates_logging.config', 'process_coordinates_%s.log' % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( ) help = "Process word coordinates for a batch by name from a batch list file" args = '<batch_list_filename>' def handle(self, batch_list_filename, *args, **options): if len(args)!=0: raise CommandError('Usage is process_coordinates %s' % self.args) loader = batch_loader.BatchLoader() batch_list = file(batch_list_filename) _logger.info("batch_list_filename: %s" % batch_list_filename) for line in batch_list: batch_name = line.strip() _logger.info("batch_name: %s" % batch_name)
from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.index import index_pages configure_logging("index_pages_logging.config", "index_pages.log") class Command(BaseCommand): def handle(self, **options): index_pages()
import logging import os from django.core import management from django.core.management.base import BaseCommand from chronam.core import models from chronam.core.holding_loader import HoldingLoader from chronam.core.management.commands import configure_logging from chronam.core.utils.utils import validate_bib_dir configure_logging('load_holdings_logging.config', 'load_holdings.log') _logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Load a holdings records after title records are all loaded" args = '<location of holdings directory>' bib_in_settings = validate_bib_dir() if bib_in_settings: default_location = bib_in_settings + '/holdings' else: default_location = None def handle(self, holdings_source=default_location, *args, **options): if not os.path.exists(holdings_source): _logger.error("There is no valid holdings source folder defined.") set_holdings = ['To load holdings - Add a folder called "holdings"', 'to the bib directory that is set in settings',
from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.models import Essay from chronam.core.essay_loader import purge_essay configure_logging("purge_essays.config", 'purge_essays.log') class Command(BaseCommand): help = "purge all the essays" def handle(self, *args, **options): for essay in Essay.objects.all(): purge_essay(essay.essay_editor_url)
from solr import SolrConnection from django.conf import settings from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.batch_loader import BatchLoader from chronam.core import batch_loader from chronam.core.management.commands import configure_logging from chronam.core import title_loader from chronam.core.index import index_titles from chronam.core.models import Title from slackclient import SlackClient configure_logging('load_dlg_batches_logging.config', 'load_dlg_batches_%s.log' % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--skip-process-ocr', action='store_false', dest='process_ocr', default=True, help='Do not generate ocr, and index'), make_option('--skip-process-coordinates', action='store_false', dest='process_coordinates', default=True,
import sys import simplejson as json import os import logging from django.conf import settings from django.core.management.base import BaseCommand, CommandError from chronam.core.models import Title, FundingSource, NewspaperType from django.db.utils import IntegrityError from chronam.core.management.commands import configure_logging configure_logging('sync_extra_metadata.config', 'sync_extra_metadata_%s.log' % os.getpid()) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): help = 'Adds custom GHNP metadata to title records by LCCN' def handle(self, *args, **options): files_processed = 0 json_path = settings.EXTRA_METADATA # iterate through each file if not os.path.isdir(json_path): raise CommandError( 'EXTRA_METADATA path in settings.py is not a directory!') json_dir = os.listdir(json_path)
import logging from cStringIO import StringIO from optparse import make_option from django.core.management.base import BaseCommand import pymarc from chronam.core.management.commands import configure_logging from chronam.core import index from chronam.core.models import Title configure_logging("chronam_purge_titles.config", "chronam_purge_etitles.log") _log = logging.getLogger(__name__) class Command(BaseCommand): """ Management command for purging title records which have an 856 field containing a link to Chronicling America, and which appear to be records for an electronic only version of a title 245 $h == [electronic resource]. The script is careful not to purge any records that have issues attached to them. See https://rdc.lctl.gov/trac/ndnp/ticket/375 for context. If you want to see the records that will be purged use the --pretend option. """ option_list = BaseCommand.option_list + ( make_option('-p', '--pretend', dest='pretend', action='store_true'), )
import csv from optparse import make_option from time import mktime from datetime import datetime import feedparser from django.core.management.base import BaseCommand from django.conf import settings from chronam.core.management.commands import configure_logging from chronam.core.rdf import rdf_uri from chronam.core import models as m configure_logging("release.config", "release.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Updates (Resets if --reset option is used) release datetime on batches from one of following sources (in order of preference) 1. bag-info.txt, if found in the batch source 2. If path to a file is provided with the command, datetime is extracted from the file 3. current public feed 4. current server datetime" reset = make_option('--reset', action = 'store_true', dest = 'reset', default = False, help = 'reset release times to nothing before setting them again') option_list = BaseCommand.option_list + (reset, ) def handle(self, *args, **options): if options['reset']:
import logging from django.core.management.base import BaseCommand from django.conf import settings from chronam.core.essay_loader import load_essays from chronam.core.management.commands import configure_logging configure_logging("load_essays.config", 'load_essays.log') log = logging.getLogger() class Command(BaseCommand): help = "load all the essays" def handle(self, *args, **options): load_essays(settings.ESSAYS_FEED, index=True)
import os import logging from optparse import make_option from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core import models from chronam.core.management.commands import configure_logging configure_logging('diff_batches_logging.config', 'diff_batches_%s.log' % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + (make_option( '--skip-process-ocr', action='store_false', dest='process_ocr', default=True, help='Do not generate ocr, and index'), ) help = "Diff batches by name from a batch list file" args = '<batch_list_filename>' def handle(self, batch_list_filename, *args, **options): if len(args) != 0: raise CommandError('Usage is diff_batch %s' % self.args) batches = set()
import os import logging from optparse import make_option from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.batch_loader import BatchLoader, BatchLoaderException from chronam.core.management.commands import configure_logging configure_logging('load_batch_logging.config', 'load_batch_%s.log' % os.getpid()) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--skip-process-ocr', action='store_false', dest='process_ocr', default=True, help='Do not generate ocr, and index'), make_option('--skip-coordinates', action='store_false', dest='process_coordinates', default=True, help='Do not out word coordinates'), ) help = "Load a batch" args = '<batch name>'
import logging from django.core.cache import cache from django.core.management.base import BaseCommand, CommandError from chronam.core.management.commands import configure_logging configure_logging('', 'purge_django_cache.log' ) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): help = "Purge the django cache after ingest/purge of a batch" def handle(self, *args, **options): try: # delete the total pages count LOGGER.info('removing newspaper_info from cache') cache.delete('newspaper_info') # delete the advanced search title list LOGGER.info('removing titles_states from cache') cache.delete('titles_states') except Exception, e: LOGGER.exception(e) raise CommandError("unable to purge the cache. check the purge_batch_cache log for clues")
import os import logging from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.management.commands import configure_logging from chronam.core import tasks configure_logging('queue_process_coordinates.config', 'queue_process_coordinates_%s.log' % os.getpid()) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( ) help = "queue the word coordinates of a batch to be processed" args = '<batch name>' def handle(self, batch_name, *args, **options): if len(args)!=0: raise CommandError('Usage is queue_process_coordinates %s' % self.args) try: tasks.process_coordinates.delay(batch_name) except Exception, e: LOGGER.exception(e) raise CommandError("unable to process coordinates. check the queue_load_batch log for clues")
import logging from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.index import index_titles configure_logging("index_titles_logging.config", "index_titles.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): def handle(self, **options): _logger.info("indexing titles") index_titles() _logger.info("finished indexing titles")
import os import logging from optparse import make_option from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.models import Page from django.conf import settings configure_logging('fix_pages.config', 'fix_pages_%s.log' % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--save', action='store_true', dest='save', default=False, help='Save Pages after updating'), ) help = "Fix full paths on some Page records." def handle(self, *args, **options): pages = Page.objects.all() bsp = settings.BATCH_STORAGE paths_fixed = 0 for page in pages:
import logging from datetime import datetime import os from optparse import make_option from django.core.management.base import BaseCommand from chronam.core import title_loader from chronam.core.index import index_titles from chronam.core.models import Title from chronam.core.management.commands import configure_logging configure_logging('load_titles_logging.config', 'load_titles.log') _logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Load a marcxml file of title records" args = '<location of marcxml>' option_list = BaseCommand.option_list + (make_option('--skip-index', action='store_true', dest='skip_index', default=False, help="\ Skip the index process. Use this if you call this from \ another process such as 'chronam_sync'. If you call this \ directly, you don't want to use this flag. \ "), ) def __init__(self):
import logging from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.index import index_titles, index_pages configure_logging("index_logging.config", "index.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): help = "index all titles and pages ; " + \ "you may (or may not) want to zap_index before" def handle(self, **options): _logger.info("indexing titles") index_titles() _logger.info("finished indexing titles") _logger.info("indexing pages") index_pages() _logger.info("finished indexing pages")
from django.conf import settings from django.core.management import call_command from django.core.management.base import BaseCommand try: import simplejson as json except ImportError: import json from chronam import core from chronam.core import index from chronam.core.management.commands import configure_logging from chronam.core.models import Place, Title from chronam.core.utils.utils import validate_bib_dir configure_logging("title_sync_logging.config", "title_sync.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): skip_essays = make_option('--skip-essays', action='store_true', dest='skip_essays', default=False, help='Skip essay loading.') pull_title_updates = make_option('--pull-title-updates', action='store_true', dest='pull_title_updates', default=False,
import os import logging from optparse import make_option from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core import batch_loader from chronam.core.management.commands import configure_logging configure_logging("load_batches_logging.config", "load_batches_%s.log" % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option( "--skip-process-ocr", action="store_false", dest="process_ocr", default=True, help="Do not generate ocr, and index", ), make_option( "--skip-process-coordinates", action="store_false", dest="process_ocr", default=True, help="Do not write out word coordinates", ),
import os import logging from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.management.commands import configure_logging from chronam.core import tasks configure_logging('queue_purge_batch_logging.config', 'queue_purge_batch_%s.log' % os.getpid()) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( ) help = "queue a batch to be purged" args = '<batch name>' def handle(self, batch_name, *args, **options): if len(args)!=0: raise CommandError('Usage is queue_purge_batch %s' % self.args) try: tasks.purge_batch.delay(batch_name) except Exception, e: LOGGER.exception(e) raise CommandError("unable to queue purge batch. check the queue_purge_batch log for clues")
import os import csv import codecs from django.core.management.base import BaseCommand from chronam.core.management.commands import configure_logging from chronam.core.models import Institution configure_logging("load_intitutions_logging.config", "load_institutions_%s.log" % os.getpid()) """ Simple command to load institution data obtained from the MySQL database running in the MARC Standards office. "oid","orgName","altname1","altname2","altname3","altname4","orgCode","lowercode","isilCode","obsoleteOrgCode","createDate","modifiedDate","address1","address2","address3","city","stateID","zip","countryID","ID","cname","prefix","searchable" 22035,"3Com Corporation Technical Library","","","","","CStcTCC","cstctcc","US-CStcTCC","","1995-10-19 00:00:00","1995-10-19 00:00:00","5400 Bayfront Plaza","","","Santa Clara",5,"95052",210,210,"United States","US","yes" """ class Command(BaseCommand): help = 'loads institution csv data into Institution table' args = '<institution_csv_file>' def handle(self, csv_file, *args, **options): for row in unicode_csv_reader(codecs.open(csv_file, encoding='utf-8')): if row[20] != 'United States': continue i = Institution() i.code = row[7].upper() i.name = row[1]
import os import logging from optparse import make_option from django.core.management.base import BaseCommand, CommandError from chronam.core.batch_loader import BatchLoader, BatchLoaderException from chronam.core.management.commands import configure_logging configure_logging('load_batch_logging.config', 'load_batch_%s.log' % os.getpid()) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--skip-process-ocr', action='store_false', dest='process_ocr', default=True, help='Do not generate ocr, and index'), make_option('--skip-coordinates', action='store_false', dest='process_coordinates', default=True, help='Do not out word coordinates'), make_option('--in-copyright', dest='in_copyright', default=False, help='Do not make available via API'),
import os import logging from optparse import make_option from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core import models from chronam.core.management.commands import configure_logging configure_logging('diff_batches_logging.config', 'diff_batches_%s.log' % os.getpid()) _logger = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--skip-process-ocr', action='store_false', dest='process_ocr', default=True, help='Do not generate ocr, and index'), ) help = "Diff batches by name from a batch list file" args = '<batch_list_filename>' def handle(self, batch_list_filename, *args, **options): if len(args)!=0: raise CommandError('Usage is diff_batch %s' % self.args) batches = set()
from django.core.management import call_command from django.core.management.base import BaseCommand try: import simplejson as json except ImportError: import json from chronam import core from chronam.core import index from chronam.core.essay_loader import load_essays from chronam.core.management.commands import configure_logging from chronam.core.models import Place, Title configure_logging("title_sync_logging.config", "title_sync.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): skip_essays = make_option('--skip-essays', action='store_true', dest='skip_essays', default=False, help='Skip essay loading.') pull_title_updates = make_option('--pull-title-updates', action='store_true', dest='pull_title_updates', default=False, help='Pull down a new set of titles.')
import os import logging from datetime import datetime from optparse import make_option from django.core import management from django.core.management.base import BaseCommand from chronam.core import models from chronam.core import index from chronam.core.management.commands import configure_logging from chronam.core.utils.utils import validate_bib_dir configure_logging("chronam_sync_logging.config", "chronam_sync.log") _logger = logging.getLogger(__name__) class Command(BaseCommand): verbose = make_option('--verbose', action='store_true', dest='verbose', default=False, help='') skip_essays = make_option('--skip-essays', action='store_true', dest='skip_essays', default=False, help='Skip essay loading.') pull_title_updates = make_option('--pull-title-updates',
import os import logging from django.core.management.base import BaseCommand from django.core.management.base import CommandError from chronam.core.management.commands import configure_logging from chronam.core import tasks configure_logging('queue_purge_batch_logging.config', 'queue_purge_batch_%s.log' % os.getpid()) LOGGER = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + () help = "queue a batch to be purged" args = '<batch name>' def handle(self, batch_name, *args, **options): if len(args) != 0: raise CommandError('Usage is queue_purge_batch %s' % self.args) try: tasks.purge_batch.delay(batch_name) except Exception, e: LOGGER.exception(e) raise CommandError( "unable to queue purge batch. check the queue_purge_batch log for clues" )
import os import logging from optparse import make_option from django.conf import settings from django.core.management.base import BaseCommand, CommandError from solr import SolrConnection from chronam.core.batch_loader import BatchLoader, BatchLoaderException from chronam.core.management.commands import configure_logging configure_logging('purge_batches_logging.config', 'purge_batch_%s.log' % os.getpid()) log = logging.getLogger(__name__) class Command(BaseCommand): option_list = BaseCommand.option_list + (make_option( '--no-optimize', action='store_false', dest='optimize', default=True, help='Do not optimize Solr and MySQL after purge'), ) help = "Purge a batch" args = '<batch_location>' def handle(self, batch_location=None, *args, **options): if len(args) != 0:
import logging import urlparse import urllib2 import datetime from re import sub from time import time, strptime from pymarc import map_xml, record_to_xml from django.db import reset_queries from chronam.core import models from chronam.core.management.commands import configure_logging configure_logging('load_titles_logging.config', 'load_titles.log') _logger = logging.getLogger(__name__) class TitleLoader(object): def __init__(self): self.records_processed = 0 self.records_created = 0 self.records_updated = 0 self.records_deleted = 0 self.missing_lccns = 0 self.errors = 0 def load_file(self, location, skip=0): location = urlparse.urljoin("file:", location) t0 = time() times = []
import logging from cStringIO import StringIO from optparse import make_option from django.core.management.base import BaseCommand import pymarc from chronam.core.management.commands import configure_logging from chronam.core import index from chronam.core.models import Title configure_logging("chronam_purge_titles.config", "chronam_purge_etitles.log") _log = logging.getLogger(__name__) class Command(BaseCommand): """ Management command for purging title records which have an 856 field containing a link to Chronicling America, and which appear to be records for an electronic only version of a title 245 $h == [electronic resource]. The script is careful not to purge any records that have issues attached to them. See https://rdc.lctl.gov/trac/ndnp/ticket/375 for context. If you want to see the records that will be purged use the --pretend option. """ option_list = BaseCommand.option_list + (make_option( '-p', '--pretend', dest='pretend', action='store_true'), )
import logging from django.core.management.base import BaseCommand from django.conf import settings from chronam.core.essay_loader import load_essays from chronam.core.management.commands import configure_logging configure_logging("load_essays.config", "load_essays.log") log = logging.getLogger() class Command(BaseCommand): help = "load all the essays" def handle(self, *args, **options): load_essays(settings.ESSAYS_FEED, index=True)