def run_remove_entry_via_id( id_to_remove ):
""" Removes id from solr.
Triggered by utils.reindex_all_support.run_enqueue_all_index_updates(). """
log = log_helper.setup_logger()
indexer = Indexer( log )
indexer.remove_index_entry( inscription_id=id_to_remove )
return
def run_update_entry( updated_file_path ):
""" Updates solr index for a new or changed file.
Triggered by run_update_index(), and utils.reindex_all_support.run_enqueue_all_index_updates(). """
log = log_helper.setup_logger()
indexer = Indexer( log )
filename = updated_file_path.split( u'/' )[-1]
indexer.update_index_entry( filename )
return
def run_remove_entry( removed_file_path ):
""" Updates solr index for removed file.
Triggered by run_update_index(). """
log = log_helper.setup_logger()
indexer = Indexer( log )
filename = removed_file_path.split( u'/' )[-1]
indexer.remove_index_entry( filename=filename )
return
def run_call_simple_git_pull():
""" Initiates a simple git pull update.
Triggered by usep_gh_handler.reindex_all() """
log = log_helper.setup_logger()
puller = Puller( log )
puller.call_git_pull()
q.enqueue_call(
func=u'usep_gh_handler_app.utils.reindex_all_support.run_simple_copy_files',
kwargs={} )
return
def run_simple_copy_files():
""" Runs a copy and then triggers an full_re-index job.
Triggered by utils.processor.run_call_simple_git_pull(). """
log = log_helper.setup_logger()
log.debug( u'in utils.reindex_all_support.run_simple_copy_files(); starting' )
copier = Copier( log )
copier.copy_files()
q.enqueue_call(
func=u'usep_gh_handler_app.utils.reindex_all_support.run_start_reindex_all',
kwargs={} )
return
def run_build_solr_remove_list( inscriptions ):
""" Builds a list of inscription_ids to remove from solr.
Triggered by run_start_reindex_all(). """
assert inscriptions.keys() == [ u'inscriptions' ]
log = log_helper.setup_logger()
solr_id_checker = SolrIdChecker( log )
( inscriptions_to_index, ids_to_remove ) = solr_id_checker.build_orphaned_ids( inscriptions[u'inscriptions'] )
q.enqueue_call(
func=u'usep_gh_handler_app.utils.reindex_all_support.run_enqueue_all_index_updates',
kwargs={u'inscriptions_to_index': inscriptions_to_index, u'ids_to_remove': ids_to_remove} )
return
def run_update_index( files_updated, files_removed ):
""" Creates index jobs (doesn't actually call Indexer() directly.
Triggered by utils.processor.run_xinclude_updater(). """
log = log_helper.setup_logger()
indexer = Indexer( log )
for updated_file_path in files_updated:
if indexer.check_updated_file_path( updated_file_path ):
q.enqueue_call(
func=u'usep_gh_handler_app.utils.indexer.run_update_entry', kwargs={u'updated_file_path': updated_file_path} )
for removed_file_path in files_removed:
if indexer.check_removed_file_path( removed_file_path ):
q.enqueue_call(
func=u'usep_gh_handler_app.utils.indexer.run_remove_entry', kwargs={u'removed_file_path': removed_file_path} )
return
def run_xinclude_updater( files_to_update, files_to_remove ):
""" Updates the three <xi:include href="../path/inscription.xml"> hrefs in each inscription file.
Reason is that the folder structure as exists for editors and on github is slightly different than in web-app.
Triggered bu utils.processor.run_copy_files(). """
log = log_helper.setup_logger()
log.debug( u'in utils.processor.run_call_xinclude_replacer(); starting' )
assert type( files_to_update ) == list; assert type( files_to_remove ) == list
xinclude_updater = XIncludeUpdater( log )
xinclude_updater.update_xinclude_references()
log.debug( u'in processor.run_call_xinclude_updater(); enqueuing next job' )
q.enqueue_call(
func=u'usep_gh_handler_app.utils.indexer.run_update_index',
kwargs={u'files_updated': files_to_update, u'files_removed': files_to_remove} )
return
def run_copy_files( files_to_update, files_to_remove ):
""" Runs a copy and then triggers an index job.
Incoming data not for copying, but to pass to indexer.
Triggered by utils.processor.run_call_git_pull(). """
log = log_helper.setup_logger()
log.debug( u'in utils.processor.run_copy_files(); starting' )
assert type( files_to_update ) == list; assert type( files_to_remove ) == list
log.debug( u'in utils.processor.run_copy_files(); files_to_update, `%s`' % pprint.pformat(files_to_update) )
log.debug( u'in utils.processor.run_copy_files(); files_to_remove, `%s`' % pprint.pformat(files_to_remove) )
copier = Copier( log )
copier.copy_files()
q.enqueue_call(
func=u'usep_gh_handler_app.utils.processor.run_xinclude_updater',
kwargs={u'files_to_update': files_to_update, u'files_to_remove': files_to_remove} )
return
def run_call_git_pull( files_to_process ):
""" Initiates a git pull update.
Spawns a call to Processor.process_file() for each result found.
Triggered by usep_gh_handler.handle_github_push(). """
log = log_helper.setup_logger()
assert sorted( files_to_process.keys() ) == [ u'files_removed', u'files_updated', u'timestamp']; log.debug( u'in utils.processor.run_call_git_pull(); files_to_process, `%s`' % pprint.pformat(files_to_process) )
time.sleep( 2 ) # let any existing jobs in process finish
( puller, copier ) = ( Puller(log), Copier(log) )
puller.call_git_pull()
( files_to_update, files_to_remove ) = ( copier.get_files_to_update(files_to_process), copier.get_files_to_remove(files_to_process) )
log.debug( u'in utils.processor.run_call_git_pull(); enqueuing next job' )
q.enqueue_call(
func=u'usep_gh_handler_app.utils.processor.run_copy_files',
kwargs={u'files_to_update': files_to_update, u'files_to_remove': files_to_remove} )
return
def run_start_reindex_all():
""" Starts process by building a list of inscriptions.
Process:
- build list of all inscriptions (from file system) that need to be indexed.
- build list of all inscription_ids that need to be removed,
meaning inscription_ids that _are_ in solr and are _not_ in the file system inscriptions list.
- enqueue all add jobs
- enqueue all remove jobs
Triggered by utils.processor.run_simple_copy_files().
"""
log = log_helper.setup_logger()
filenames_builder = InscriptionFilenamesBuilder( log )
inscriptions = filenames_builder.build_inscription_filepaths()
q.enqueue_call(
func=u'usep_gh_handler_app.utils.reindex_all_support.run_build_solr_remove_list',
kwargs={u'inscriptions': inscriptions} )
return
# -*- coding: utf-8 -*-
import datetime, json, os, pprint, urlparse
import flask, redis, requests, rq
from flask.ext.basicauth import BasicAuth # http://flask-basicauth.readthedocs.org/en/latest/
from usep_gh_handler_app.utils import log_helper, reindex_all_support
from usep_gh_handler_app.utils.web_app_helper import WebAppHelper
## setup
B_AUTH_PASSWORD = unicode( os.environ[u'usep_gh__BASIC_AUTH_PASSWORD'] )
B_AUTH_USERNAME = unicode( os.environ[u'usep_gh__BASIC_AUTH_USERNAME'] )
app = flask.Flask(__name__)
log = log_helper.setup_logger()
app.config[u'BASIC_AUTH_USERNAME'] = B_AUTH_USERNAME
app.config[u'BASIC_AUTH_PASSWORD'] = B_AUTH_PASSWORD
basic_auth = BasicAuth(app)
app_helper = WebAppHelper( log )
q = rq.Queue( u'usep', connection=redis.Redis() )
@app.route( u'/reindex_all/', methods=[u'GET'] )
@basic_auth.required
def reindex_all():
""" Triggers a git-pull and a re-index of everything.
Called via admin. """
try:
log.debug( u'in usep_gh_handler.reindex_all(); starting' )
q.enqueue_call (
func=u'usep_gh_handler_app.utils.reindex_all_support.run_call_simple_git_pull',
kwargs = {} )
