def handle(self, batch_list_filename, *args, **options):
if len(args) != 0:
raise CommandError("Usage is process_coordinates %s" % self.args)
verbosity = options["verbosity"]
if verbosity > 0:
log_level = logging.DEBUG if verbosity > 1 else logging.INFO
loggers = [logging.getLogger(), logging.getLogger("chronam")]
for logger in loggers:
logger.setLevel(log_level)
for handler in logger.handlers:
handler.setLevel(log_level)
loader = batch_loader.BatchLoader()
if os.path.exists(batch_list_filename):
with open(batch_list_filename) as f:
batches = [i.strip() for i in f]
else:
batches = [batch_list_filename]
LOGGER.info("Processing %d batches from %s", len(batches),
batch_list_filename)
for batch_name in batches:
LOGGER.info("batch_name: %s" % batch_name)
loader.process_coordinates(batch_name)
def handle(self, batch_list_filename, *args, **options):
def slack(message):
sc.api_call("chat.postMessage", channel="#ghnp", text=message)
def log(message):
_logger.info(message.replace('`', ''))
def update(message):
slack(message)
log(message)
if len(args) != 0:
raise CommandError('Usage is load_batch %s' % self.args)
added_batches = []
failed_batches = []
skipped_batches = []
processed = 0
start = datetime.now()
sc = SlackClient(settings.SLACK_KEY)
loader = batch_loader.BatchLoader(
process_ocr=options['process_ocr'],
process_coordinates=options['process_coordinates'])
# get legit batch names
with open(batch_list_filename) as f:
batch_names_from_file = f.readlines()
batch_names_from_file[:] = [
batch_name for batch_name in batch_names_from_file
if batch_name.strip()
]
count = len(batch_names_from_file)
update('Loading `%s` Batches from file: `%s`' %
(count, batch_list_filename))
for line in batch_names_from_file:
batch_start = datetime.now()
processed += 1
batch_name = line.strip()
try:
if Batch.objects.filter(name=batch_name).count() != 0:
skipped_batches.append(batch_name)
continue
update('Loading batch `%s` of `%s`: `%s`' %
(processed, count, batch_name))
batch = loader.load_batch(batch_name, strict=False)
added_batches.append(batch_name)
update('`%s` loaded in `%s`.' %
(batch_name, datetime.now() - batch_start))
except Exception, e:
update('`%s` failed to load. Error: `%s`.' %
(batch_name, str(e)))
failed_batches.append(batch_name)
continue
def handle(self, batch_list_filename, *args, **options):
if len(args) != 0:
raise CommandError('Usage is load_batch %s' % self.args)
loader = batch_loader.BatchLoader()
loader.PROCESS_OCR = options['process_ocr']
loader.PROCESS_COORDINATES = options['process_coordinates']
batch_list = open(batch_list_filename)
self.stdout.write("batch_list_filename: %s" % batch_list_filename)
for line in batch_list:
batch_name = line.strip()
self.stdout.write("batch_name: %s" % batch_name)
loader.load_batch(batch_name, strict=False)
def handle(self, batch_list_filename, *args, **options):
if len(args)!=0:
raise CommandError('Usage is process_coordinates %s' % self.args)
loader = batch_loader.BatchLoader()
batch_list = file(batch_list_filename)
_logger.info("batch_list_filename: %s" % batch_list_filename)
for line in batch_list:
batch_name = line.strip()
_logger.info("batch_name: %s" % batch_name)
parts = batch_name.split("_")
if len(parts)==4:
loader.process_coordinates(batch_name)
else:
_logger.warning("invalid batch name '%s'" % batch_name)
def handle(self, batch_list_filename, *args, **options):
if len(args) != 0:
raise CommandError('Usage is load_batch %s' % self.args)
loader = batch_loader.BatchLoader(
process_ocr=options['process_ocr'],
process_coordinates=options['process_coordinates'])
batch_list = file(batch_list_filename)
_logger.info("batch_list_filename: %s" % batch_list_filename)
for line in batch_list:
batch_name = line.strip()
_logger.info("batch_name: %s" % batch_name)
parts = batch_name.split("_")
if len(parts) == 4 and parts[0] == "batch":
loader.load_batch(batch_name, strict=False)
else:
_logger.warning("invalid batch name '%s'" % batch_name)
def handle(self, *args, **options):
def get_immediate_subdirectories(a_dir):
return [
name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))
]
def slack(message):
sc.api_call("chat.postMessage", channel="#ghnp", text=message)
start = datetime.now()
sc = SlackClient(settings.SLACK_KEY)
loader = BatchLoader()
new_batches_location = '/opt/chronam/data/chronamftp/new_batches/'
replacement_batches_location = '/opt/chronam/data/chronamftp/replacement_batches/'
nonlccn_location = '/opt/chronam/data/nonlccn/'
batch_drop = '/opt/chronam/data/dlg_batches/drop/'
# GET LIST OF BATCHES TO LOAD
new_batches = get_immediate_subdirectories(new_batches_location)
replacement_batches = get_immediate_subdirectories(
replacement_batches_location)
# CHECK new_batches FOR finalMARC FOLDERS
new_title_folders = []
for folder in new_batches:
if 'MARC' in folder:
new_title_folders.append(folder)
new_batches.remove(folder)
# ISSUE STARTING NOTIFICATIONS
slack(
'Starting DLG Batch Load Process! Found `%s` new batches and `%s` replacement batches available to load.'
% (len(new_batches), len(replacement_batches)))
# RUN KEVIN'S RSYNC COMMANDS, WAIT
slack('RSync of batches is starting')
start_time = time.time()
slack('Copying new batches')
subprocess.call([
'rsync -rav --progress /opt/chronam/data/chronamftp/new_batches/* /opt/chronam/data/dlg_batches/drop/'
])
slack('Copying replacement batches')
subprocess.call([
'rsync -rav --progress /opt/chronam/data/chronamftp/replacement_batches/* /opt/chronam/data/dlg_batches/drop/'
])
duration = time.time() - start_time
slack('RSync of new and replacement batches completed in %s seconds' %
duration)
# LOAD NEW TITLES IF PRESENT
if new_title_folders:
slack('Also found `%s` title MARC files to process.' %
len(new_title_folders))
for nt in new_title_folders:
for nt_f in os.listdir(os.path.join(new_batches_location, nt)):
if nt_f.endswith('.xml'):
marc_file = os.path.join(nonlccn_location, nt_f)
copyfile(os.path.join(new_batches_location, nt, nt_f),
marc_file)
title_load_results = title_loader.load(marc_file)
if title_load_results[1]:
slack('New title created from `%s`.' % nt_f)
if title_load_results[2]:
slack('Title updated from `%s`.' % nt_f)
if title_load_results[3]:
slack('Error on title load from `%s`' % nt_f)
index_titles(start)
slack('Finished loading titles.')
# PURGE REPLACEMENT BATCHES
if replacement_batches:
slack('Purging batches destined for replacement.')
for r_b in replacement_batches:
batch_to_purge = r_b.replace('ver02','ver01')\
.replace('ver03','ver02')\
.replace('ver04','ver03')\
.replace('ver05','ver04')\
.replace('ver06','ver05')\
.replace('ver07','ver06')\
.replace('ver08','ver07')
slack('Purging `%s`.' % batch_to_purge)
loader.purge_batch(batch_to_purge)
start_time = time.time()
solr = SolrConnection(settings.SOLR)
solr.optimize()
slack('Index optimize complete in `%s` seconds.' % time.time() -
start_time)
# LOAD ALL BATCHES
# start with replacement batches
final_loader = batch_loader.BatchLoader(process_ocr=True,
process_coordinates=True)
if replacement_batches:
replace_start = time.time()
for replacement in replacement_batches:
final_loader.load_batch('drop/%s' % replacement, strict=False)
slack('Loaded replacement batch `%s`.' % replacement)
slack('All replacement batches loaded in `%s` seconds.' %
time.time() - replace_start)
# load new batches
if new_batches:
new_start = time.time()
for new in new_batches:
final_loader.load_batch('drop/%s' % new, strict=False)
slack('Loaded new batch `%s`.' % new)
slack('All new batches loaded in `%s` seconds.' % time.time() -
new_start)
slack('Batch loading job complete!')