import logging
from django.core.cache import cache
from django.core.management.base import BaseCommand, CommandError
from chronam.core.management.commands import configure_logging
configure_logging('', 'purge_django_cache.log' )
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Purge the django cache after ingest/purge of a batch"
def handle(self, *args, **options):
try:
# delete the total pages count
LOGGER.info('removing newspaper_info from cache')
cache.delete('newspaper_info')
# delete the advanced search title list
LOGGER.info('removing titles_states from cache')
cache.delete('titles_states')
# Delete the fulltext range so the search form is up-to-date
LOGGER.info('Removing fulltext_range from cache')
cache.delete('fulltext_range')
except Exception, e:
import os
import logging
from optparse import make_option
from django.conf import settings
from django.db import connection
from django.core.management.base import BaseCommand, CommandError
from solr import SolrConnection
from chronam.core.batch_loader import BatchLoader, BatchLoaderException
from chronam.core.management.commands import configure_logging
configure_logging('purge_batches_logging.config',
'purge_batch_%s.log' % os.getpid())
log = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--no-optimize',
action='store_false',
dest='optimize', default=True,
help='Do not optimize Solr and MySQL after purge'),
)
help = "Purge a batch"
args = '<batch_location>'
def handle(self, batch_location=None, *args, **options):
import logging
import urllib2
from django.core.management.base import BaseCommand
from django.db import reset_queries
from rdflib import Namespace, ConjunctiveGraph, URIRef
try:
import simplejson as json
except ImportError:
import json
from chronam.core import models
from chronam.core.management.commands import configure_logging
configure_logging("chronam_link_places.config", "chronam_link_places.log")
_logger = logging.getLogger(__name__)
geo = Namespace('http://www.w3.org/2003/01/geo/wgs84_pos#')
owl = Namespace('http://www.w3.org/2002/07/owl#')
dbpedia = Namespace('http://dbpedia.org/ontology/')
class Command(BaseCommand):
def handle(self, **options):
_logger.debug("linking places")
for place in models.Place.objects.filter(dbpedia__isnull=True):
if not place.city or not place.state:
continue
# formulate a dbpedia place uri
import os
import logging
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.models import Place, Region
configure_logging('refine_places_logging.config',
'refine_places_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + ()
help = "Parse place names and update city, county and state values. Also add region if possible."
def handle(self, *args, **options):
metro_atlanta_counties = [
'Clayton', 'Cobb', 'DeKalb', 'Fulton', 'Gwinnett', 'Henry',
'Rockdale'
]
north_georgia_counties = [
'Banks', 'Barrow', 'Bartow', 'Catoosa', 'Chattooga', 'Cherokee',
'Clarke', 'Dade', 'Dawson', 'Elbert', 'Fannin', 'Floyd', 'Forsyth',
'Franklin', 'Gilmer', 'Gordon', 'Habersham', 'Hall', 'Hart',
'Jackson', 'Lumpkin', 'Madison', 'Morgan', 'Murray', 'Newton',
'Oconee', 'Oglethorpe', 'Pickens', 'Polk', 'Rabun', 'Stephens',
'Towns', 'Union', 'Walker', 'Walton', 'White', 'Whitfield'
import os
import logging
from datetime import datetime
from optparse import make_option
from django.core import management
from django.core.management.base import BaseCommand
from chronam.core import models
from chronam.core import index
from chronam.core.management.commands import configure_logging
from chronam.core.utils.utils import validate_bib_dir
configure_logging("chronam_sync_logging.config", "chronam_sync.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
verbose = make_option('--verbose',
action='store_true',
dest='verbose',
default=False,
help='')
skip_essays = make_option('--skip-essays',
action='store_true',
dest='skip_essays',
default=False,
help='Skip essay loading.')
pull_title_updates = make_option('--pull-title-updates',
action='store_true',
import logging
from django.core import management
from django.core.management.base import BaseCommand
from chronam.core import models
from chronam.core.holding_loader import HoldingLoader
from chronam.core.management.commands import configure_logging
configure_logging('load_holdings_logging.config', 'load_holdings.log')
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Load a holdings records after title records are all loaded"
args = '<location of holdings directory>'
def handle(self, holdings_source, *args, **options):
# First we want to make sure that our material types are up to date
material_types = models.MaterialType.objects.all()
[m.delete() for m in material_types]
management.call_command('loaddata', 'material_types.json')
holding_loader = HoldingLoader()
holding_loader.main(holdings_source)
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.management.commands import configure_logging
from chronam.core import tasks
configure_logging('poll_cts_logging.config',
'poll_cts_%s.log' % os.getpid())
log = logging.getLogger(__name__)
class Command(BaseCommand):
help = "manual command to load new batches from cts"
def handle(self, *args, **options):
try:
tasks.poll_cts.apply()
except Exception, e:
log.exception(e)
raise CommandError("unable to load batches from cts")
import os
import csv
import codecs
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.models import Institution
configure_logging("load_intitutions_logging.config", "load_institutions.log")
"""
Loads the institutions based on a CSV file in the form of:
<code>, <name>, <city>, <state>, <zip>
"""
class Command(BaseCommand):
help = 'loads institution csv data into Institution table'
args = '<institution_csv_file>'
def handle(self, csv_file, *args, **options):
for row in unicode_csv_reader(codecs.open(csv_file, encoding='utf-8')):
i = Institution()
i.code = row[0].upper()
i.name = row[1]
i.address1 = ""
i.address2 = ""
i.city = row[2]
i.state = row[3]
i.zip = row[4]
i.save()
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.management.commands import configure_logging
from chronam.core import tasks
configure_logging('poll_purge_logging.config',
'poll_purge_%s.log' % os.getpid())
log = logging.getLogger(__name__)
class Command(BaseCommand):
help = "manual command to process purge_batch requests from CTS"
def handle(self, *args, **options):
try:
tasks.poll_purge.apply()
except Exception, e:
log.exception(e)
raise CommandError("Unable to purge batches")
import logging
from datetime import datetime
from optparse import make_option
from django.core.management.base import BaseCommand
from django.conf import settings
from chronam.core import title_pull
from chronam.core.management.commands import configure_logging
configure_logging('pull_titles_logging.config', 'pull_titles.log')
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Retrieve a fresh pull of titles from OCLC. \
#TODO: add a list of example commands."
args = ''
#TODO: Remove default from lccn
option_list = BaseCommand.option_list + (
make_option('-l', '--lccn',
action='store',
dest='lccn',
default=None,
help="Pass a specific lccn to pull down updates from Worldcat."),
make_option('-o', '--oclc',
action='store',
dest='oclc',
default=None,
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core import batch_loader
from chronam.core.management.commands import configure_logging
configure_logging('process_coordinates_logging.config',
'process_coordinates_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
)
help = "Process word coordinates for a batch by name from a batch list file"
args = '<batch_list_filename>'
def handle(self, batch_list_filename, *args, **options):
if len(args)!=0:
raise CommandError('Usage is process_coordinates %s' % self.args)
loader = batch_loader.BatchLoader()
batch_list = file(batch_list_filename)
_logger.info("batch_list_filename: %s" % batch_list_filename)
for line in batch_list:
batch_name = line.strip()
_logger.info("batch_name: %s" % batch_name)
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.index import index_pages
configure_logging("index_pages_logging.config", "index_pages.log")
class Command(BaseCommand):
def handle(self, **options):
index_pages()
import logging
import os
from django.core import management
from django.core.management.base import BaseCommand
from chronam.core import models
from chronam.core.holding_loader import HoldingLoader
from chronam.core.management.commands import configure_logging
from chronam.core.utils.utils import validate_bib_dir
configure_logging('load_holdings_logging.config', 'load_holdings.log')
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Load a holdings records after title records are all loaded"
args = '<location of holdings directory>'
bib_in_settings = validate_bib_dir()
if bib_in_settings:
default_location = bib_in_settings + '/holdings'
else:
default_location = None
def handle(self, holdings_source=default_location, *args, **options):
if not os.path.exists(holdings_source):
_logger.error("There is no valid holdings source folder defined.")
set_holdings = ['To load holdings - Add a folder called "holdings"',
'to the bib directory that is set in settings',
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.models import Essay
from chronam.core.essay_loader import purge_essay
configure_logging("purge_essays.config", 'purge_essays.log')
class Command(BaseCommand):
help = "purge all the essays"
def handle(self, *args, **options):
for essay in Essay.objects.all():
purge_essay(essay.essay_editor_url)
from solr import SolrConnection
from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.batch_loader import BatchLoader
from chronam.core import batch_loader
from chronam.core.management.commands import configure_logging
from chronam.core import title_loader
from chronam.core.index import index_titles
from chronam.core.models import Title
from slackclient import SlackClient
configure_logging('load_dlg_batches_logging.config',
'load_dlg_batches_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--skip-process-ocr',
action='store_false',
dest='process_ocr',
default=True,
help='Do not generate ocr, and index'),
make_option('--skip-process-coordinates',
action='store_false',
dest='process_coordinates',
default=True,
import sys
import simplejson as json
import os
import logging
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from chronam.core.models import Title, FundingSource, NewspaperType
from django.db.utils import IntegrityError
from chronam.core.management.commands import configure_logging
configure_logging('sync_extra_metadata.config',
'sync_extra_metadata_%s.log' % os.getpid())
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'Adds custom GHNP metadata to title records by LCCN'
def handle(self, *args, **options):
files_processed = 0
json_path = settings.EXTRA_METADATA
# iterate through each file
if not os.path.isdir(json_path):
raise CommandError(
'EXTRA_METADATA path in settings.py is not a directory!')
json_dir = os.listdir(json_path)
import logging
from cStringIO import StringIO
from optparse import make_option
from django.core.management.base import BaseCommand
import pymarc
from chronam.core.management.commands import configure_logging
from chronam.core import index
from chronam.core.models import Title
configure_logging("chronam_purge_titles.config", "chronam_purge_etitles.log")
_log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Management command for purging title records which have an 856 field
containing a link to Chronicling America, and which appear to be records
for an electronic only version of a title 245 $h == [electronic resource].
The script is careful not to purge any records that have issues attached
to them. See https://rdc.lctl.gov/trac/ndnp/ticket/375 for context.
If you want to see the records that will be purged use the --pretend
option.
"""
option_list = BaseCommand.option_list + (
make_option('-p', '--pretend', dest='pretend', action='store_true'),
)
import csv
from optparse import make_option
from time import mktime
from datetime import datetime
import feedparser
from django.core.management.base import BaseCommand
from django.conf import settings
from chronam.core.management.commands import configure_logging
from chronam.core.rdf import rdf_uri
from chronam.core import models as m
configure_logging("release.config", "release.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Updates (Resets if --reset option is used) release datetime on batches from one of following sources (in order of preference) 1. bag-info.txt, if found in the batch source 2. If path to a file is provided with the command, datetime is extracted from the file 3. current public feed 4. current server datetime"
reset = make_option('--reset',
action = 'store_true',
dest = 'reset',
default = False,
help = 'reset release times to nothing before setting them again')
option_list = BaseCommand.option_list + (reset, )
def handle(self, *args, **options):
if options['reset']:
import logging
from django.core.management.base import BaseCommand
from django.conf import settings
from chronam.core.essay_loader import load_essays
from chronam.core.management.commands import configure_logging
configure_logging("load_essays.config", 'load_essays.log')
log = logging.getLogger()
class Command(BaseCommand):
help = "load all the essays"
def handle(self, *args, **options):
load_essays(settings.ESSAYS_FEED, index=True)
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.index import index_pages
configure_logging("index_pages_logging.config", "index_pages.log")
class Command(BaseCommand):
def handle(self, **options):
index_pages()
import os
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core import models
from chronam.core.management.commands import configure_logging
configure_logging('diff_batches_logging.config',
'diff_batches_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (make_option(
'--skip-process-ocr',
action='store_false',
dest='process_ocr',
default=True,
help='Do not generate ocr, and index'), )
help = "Diff batches by name from a batch list file"
args = '<batch_list_filename>'
def handle(self, batch_list_filename, *args, **options):
if len(args) != 0:
raise CommandError('Usage is diff_batch %s' % self.args)
batches = set()
import os
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.batch_loader import BatchLoader, BatchLoaderException
from chronam.core.management.commands import configure_logging
configure_logging('load_batch_logging.config',
'load_batch_%s.log' % os.getpid())
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--skip-process-ocr',
action='store_false',
dest='process_ocr', default=True,
help='Do not generate ocr, and index'),
make_option('--skip-coordinates',
action='store_false',
dest='process_coordinates', default=True,
help='Do not out word coordinates'),
)
help = "Load a batch"
args = '<batch name>'
import logging
from django.core.cache import cache
from django.core.management.base import BaseCommand, CommandError
from chronam.core.management.commands import configure_logging
configure_logging('', 'purge_django_cache.log' )
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Purge the django cache after ingest/purge of a batch"
def handle(self, *args, **options):
try:
# delete the total pages count
LOGGER.info('removing newspaper_info from cache')
cache.delete('newspaper_info')
# delete the advanced search title list
LOGGER.info('removing titles_states from cache')
cache.delete('titles_states')
except Exception, e:
LOGGER.exception(e)
raise CommandError("unable to purge the cache. check the purge_batch_cache log for clues")
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.management.commands import configure_logging
from chronam.core import tasks
configure_logging('queue_process_coordinates.config',
'queue_process_coordinates_%s.log' % os.getpid())
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
)
help = "queue the word coordinates of a batch to be processed"
args = '<batch name>'
def handle(self, batch_name, *args, **options):
if len(args)!=0:
raise CommandError('Usage is queue_process_coordinates %s' % self.args)
try:
tasks.process_coordinates.delay(batch_name)
except Exception, e:
LOGGER.exception(e)
raise CommandError("unable to process coordinates. check the queue_load_batch log for clues")
import logging
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.index import index_titles
configure_logging("index_titles_logging.config", "index_titles.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
def handle(self, **options):
_logger.info("indexing titles")
index_titles()
_logger.info("finished indexing titles")
import os
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.models import Page
from django.conf import settings
configure_logging('fix_pages.config',
'fix_pages_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--save',
action='store_true',
dest='save', default=False,
help='Save Pages after updating'),
)
help = "Fix full paths on some Page records."
def handle(self, *args, **options):
pages = Page.objects.all()
bsp = settings.BATCH_STORAGE
paths_fixed = 0
for page in pages:
import logging
from datetime import datetime
import os
from optparse import make_option
from django.core.management.base import BaseCommand
from chronam.core import title_loader
from chronam.core.index import index_titles
from chronam.core.models import Title
from chronam.core.management.commands import configure_logging
configure_logging('load_titles_logging.config', 'load_titles.log')
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Load a marcxml file of title records"
args = '<location of marcxml>'
option_list = BaseCommand.option_list + (make_option('--skip-index',
action='store_true',
dest='skip_index',
default=False,
help="\
Skip the index process. Use this if you call this from \
another process such as 'chronam_sync'. If you call this \
directly, you don't want to use this flag. \
"), )
def __init__(self):
import logging
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.index import index_titles, index_pages
configure_logging("index_logging.config", "index.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "index all titles and pages ; " + \
"you may (or may not) want to zap_index before"
def handle(self, **options):
_logger.info("indexing titles")
index_titles()
_logger.info("finished indexing titles")
_logger.info("indexing pages")
index_pages()
_logger.info("finished indexing pages")
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand
try:
import simplejson as json
except ImportError:
import json
from chronam import core
from chronam.core import index
from chronam.core.management.commands import configure_logging
from chronam.core.models import Place, Title
from chronam.core.utils.utils import validate_bib_dir
configure_logging("title_sync_logging.config", "title_sync.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
skip_essays = make_option('--skip-essays',
action='store_true',
dest='skip_essays',
default=False,
help='Skip essay loading.')
pull_title_updates = make_option('--pull-title-updates',
action='store_true',
dest='pull_title_updates',
default=False,
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core import batch_loader
from chronam.core.management.commands import configure_logging
configure_logging('process_coordinates_logging.config',
'process_coordinates_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
)
help = "Process word coordinates for a batch by name from a batch list file"
args = '<batch_list_filename>'
def handle(self, batch_list_filename, *args, **options):
if len(args)!=0:
raise CommandError('Usage is process_coordinates %s' % self.args)
loader = batch_loader.BatchLoader()
batch_list = file(batch_list_filename)
_logger.info("batch_list_filename: %s" % batch_list_filename)
for line in batch_list:
batch_name = line.strip()
_logger.info("batch_name: %s" % batch_name)
import os
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core import batch_loader
from chronam.core.management.commands import configure_logging
configure_logging("load_batches_logging.config", "load_batches_%s.log" % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option(
"--skip-process-ocr",
action="store_false",
dest="process_ocr",
default=True,
help="Do not generate ocr, and index",
),
make_option(
"--skip-process-coordinates",
action="store_false",
dest="process_ocr",
default=True,
help="Do not write out word coordinates",
),
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.management.commands import configure_logging
from chronam.core import tasks
configure_logging('queue_purge_batch_logging.config',
'queue_purge_batch_%s.log' % os.getpid())
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
)
help = "queue a batch to be purged"
args = '<batch name>'
def handle(self, batch_name, *args, **options):
if len(args)!=0:
raise CommandError('Usage is queue_purge_batch %s' % self.args)
try:
tasks.purge_batch.delay(batch_name)
except Exception, e:
LOGGER.exception(e)
raise CommandError("unable to queue purge batch. check the queue_purge_batch log for clues")
import os
import csv
import codecs
from django.core.management.base import BaseCommand
from chronam.core.management.commands import configure_logging
from chronam.core.models import Institution
configure_logging("load_intitutions_logging.config",
"load_institutions_%s.log" % os.getpid())
"""
Simple command to load institution data obtained from the MySQL database
running in the MARC Standards office.
"oid","orgName","altname1","altname2","altname3","altname4","orgCode","lowercode","isilCode","obsoleteOrgCode","createDate","modifiedDate","address1","address2","address3","city","stateID","zip","countryID","ID","cname","prefix","searchable"
22035,"3Com Corporation Technical Library","","","","","CStcTCC","cstctcc","US-CStcTCC","","1995-10-19 00:00:00","1995-10-19 00:00:00","5400 Bayfront Plaza","","","Santa Clara",5,"95052",210,210,"United States","US","yes"
"""
class Command(BaseCommand):
help = 'loads institution csv data into Institution table'
args = '<institution_csv_file>'
def handle(self, csv_file, *args, **options):
for row in unicode_csv_reader(codecs.open(csv_file, encoding='utf-8')):
if row[20] != 'United States':
continue
i = Institution()
i.code = row[7].upper()
i.name = row[1]
import os
import logging
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from chronam.core.batch_loader import BatchLoader, BatchLoaderException
from chronam.core.management.commands import configure_logging
configure_logging('load_batch_logging.config',
'load_batch_%s.log' % os.getpid())
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--skip-process-ocr',
action='store_false',
dest='process_ocr',
default=True,
help='Do not generate ocr, and index'),
make_option('--skip-coordinates',
action='store_false',
dest='process_coordinates',
default=True,
help='Do not out word coordinates'),
make_option('--in-copyright',
dest='in_copyright',
default=False,
help='Do not make available via API'),
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.management.commands import configure_logging
from chronam.core import tasks
configure_logging('poll_cts_logging.config', 'poll_cts_%s.log' % os.getpid())
log = logging.getLogger(__name__)
class Command(BaseCommand):
help = "manual command to load new batches from cts"
def handle(self, *args, **options):
try:
tasks.poll_cts.apply()
except Exception, e:
log.exception(e)
raise CommandError("unable to load batches from cts")
import os
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core import models
from chronam.core.management.commands import configure_logging
configure_logging('diff_batches_logging.config',
'diff_batches_%s.log' % os.getpid())
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--skip-process-ocr',
action='store_false',
dest='process_ocr', default=True,
help='Do not generate ocr, and index'),
)
help = "Diff batches by name from a batch list file"
args = '<batch_list_filename>'
def handle(self, batch_list_filename, *args, **options):
if len(args)!=0:
raise CommandError('Usage is diff_batch %s' % self.args)
batches = set()
from django.core.management import call_command
from django.core.management.base import BaseCommand
try:
import simplejson as json
except ImportError:
import json
from chronam import core
from chronam.core import index
from chronam.core.essay_loader import load_essays
from chronam.core.management.commands import configure_logging
from chronam.core.models import Place, Title
configure_logging("title_sync_logging.config", "title_sync.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
skip_essays = make_option('--skip-essays',
action='store_true',
dest='skip_essays',
default=False,
help='Skip essay loading.')
pull_title_updates = make_option('--pull-title-updates',
action='store_true',
dest='pull_title_updates',
default=False,
help='Pull down a new set of titles.')
import os
import logging
from datetime import datetime
from optparse import make_option
from django.core import management
from django.core.management.base import BaseCommand
from chronam.core import models
from chronam.core import index
from chronam.core.management.commands import configure_logging
from chronam.core.utils.utils import validate_bib_dir
configure_logging("chronam_sync_logging.config", "chronam_sync.log")
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
verbose = make_option('--verbose',
action='store_true',
dest='verbose',
default=False,
help='')
skip_essays = make_option('--skip-essays',
action='store_true',
dest='skip_essays',
default=False,
help='Skip essay loading.')
pull_title_updates = make_option('--pull-title-updates',
import logging
from datetime import datetime
from optparse import make_option
from django.core.management.base import BaseCommand
from django.conf import settings
from chronam.core import title_pull
from chronam.core.management.commands import configure_logging
configure_logging('pull_titles_logging.config', 'pull_titles.log')
_logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Retrieve a fresh pull of titles from OCLC. \
#TODO: add a list of example commands."
args = ''
#TODO: Remove default from lccn
option_list = BaseCommand.option_list + (
make_option('-l', '--lccn',
action='store',
dest='lccn',
default=None,
help="Pass a specific lccn to pull down updates from Worldcat."),
make_option('-o', '--oclc',
action='store',
dest='oclc',
default=None,
import os
import logging
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from chronam.core.management.commands import configure_logging
from chronam.core import tasks
configure_logging('queue_purge_batch_logging.config',
'queue_purge_batch_%s.log' % os.getpid())
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + ()
help = "queue a batch to be purged"
args = '<batch name>'
def handle(self, batch_name, *args, **options):
if len(args) != 0:
raise CommandError('Usage is queue_purge_batch %s' % self.args)
try:
tasks.purge_batch.delay(batch_name)
except Exception, e:
LOGGER.exception(e)
raise CommandError(
"unable to queue purge batch. check the queue_purge_batch log for clues"
)
import os
import logging
from optparse import make_option
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from solr import SolrConnection
from chronam.core.batch_loader import BatchLoader, BatchLoaderException
from chronam.core.management.commands import configure_logging
configure_logging('purge_batches_logging.config',
'purge_batch_%s.log' % os.getpid())
log = logging.getLogger(__name__)
class Command(BaseCommand):
option_list = BaseCommand.option_list + (make_option(
'--no-optimize',
action='store_false',
dest='optimize',
default=True,
help='Do not optimize Solr and MySQL after purge'), )
help = "Purge a batch"
args = '<batch_location>'
def handle(self, batch_location=None, *args, **options):
if len(args) != 0:
import logging
import urlparse
import urllib2
import datetime
from re import sub
from time import time, strptime
from pymarc import map_xml, record_to_xml
from django.db import reset_queries
from chronam.core import models
from chronam.core.management.commands import configure_logging
configure_logging('load_titles_logging.config', 'load_titles.log')
_logger = logging.getLogger(__name__)
class TitleLoader(object):
def __init__(self):
self.records_processed = 0
self.records_created = 0
self.records_updated = 0
self.records_deleted = 0
self.missing_lccns = 0
self.errors = 0
def load_file(self, location, skip=0):
location = urlparse.urljoin("file:", location)
t0 = time()
times = []
import logging
from cStringIO import StringIO
from optparse import make_option
from django.core.management.base import BaseCommand
import pymarc
from chronam.core.management.commands import configure_logging
from chronam.core import index
from chronam.core.models import Title
configure_logging("chronam_purge_titles.config", "chronam_purge_etitles.log")
_log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Management command for purging title records which have an 856 field
containing a link to Chronicling America, and which appear to be records
for an electronic only version of a title 245 $h == [electronic resource].
The script is careful not to purge any records that have issues attached
to them. See https://rdc.lctl.gov/trac/ndnp/ticket/375 for context.
If you want to see the records that will be purged use the --pretend
option.
"""
option_list = BaseCommand.option_list + (make_option(
'-p', '--pretend', dest='pretend', action='store_true'), )
import logging
from django.core.management.base import BaseCommand
from django.conf import settings
from chronam.core.essay_loader import load_essays
from chronam.core.management.commands import configure_logging
configure_logging("load_essays.config", "load_essays.log")
log = logging.getLogger()
class Command(BaseCommand):
help = "load all the essays"
def handle(self, *args, **options):
load_essays(settings.ESSAYS_FEED, index=True)