def run_remove_entry_via_id( id_to_remove ): """ Removes id from solr. Triggered by utils.reindex_all_support.run_enqueue_all_index_updates(). """ log = log_helper.setup_logger() indexer = Indexer( log ) indexer.remove_index_entry( inscription_id=id_to_remove ) return
def run_update_entry( updated_file_path ): """ Updates solr index for a new or changed file. Triggered by run_update_index(), and utils.reindex_all_support.run_enqueue_all_index_updates(). """ log = log_helper.setup_logger() indexer = Indexer( log ) filename = updated_file_path.split( u'/' )[-1] indexer.update_index_entry( filename ) return
def run_remove_entry( removed_file_path ): """ Updates solr index for removed file. Triggered by run_update_index(). """ log = log_helper.setup_logger() indexer = Indexer( log ) filename = removed_file_path.split( u'/' )[-1] indexer.remove_index_entry( filename=filename ) return
def run_call_simple_git_pull(): """ Initiates a simple git pull update. Triggered by usep_gh_handler.reindex_all() """ log = log_helper.setup_logger() puller = Puller( log ) puller.call_git_pull() q.enqueue_call( func=u'usep_gh_handler_app.utils.reindex_all_support.run_simple_copy_files', kwargs={} ) return
def run_simple_copy_files(): """ Runs a copy and then triggers an full_re-index job. Triggered by utils.processor.run_call_simple_git_pull(). """ log = log_helper.setup_logger() log.debug( u'in utils.reindex_all_support.run_simple_copy_files(); starting' ) copier = Copier( log ) copier.copy_files() q.enqueue_call( func=u'usep_gh_handler_app.utils.reindex_all_support.run_start_reindex_all', kwargs={} ) return
def run_build_solr_remove_list( inscriptions ): """ Builds a list of inscription_ids to remove from solr. Triggered by run_start_reindex_all(). """ assert inscriptions.keys() == [ u'inscriptions' ] log = log_helper.setup_logger() solr_id_checker = SolrIdChecker( log ) ( inscriptions_to_index, ids_to_remove ) = solr_id_checker.build_orphaned_ids( inscriptions[u'inscriptions'] ) q.enqueue_call( func=u'usep_gh_handler_app.utils.reindex_all_support.run_enqueue_all_index_updates', kwargs={u'inscriptions_to_index': inscriptions_to_index, u'ids_to_remove': ids_to_remove} ) return
def run_update_index( files_updated, files_removed ): """ Creates index jobs (doesn't actually call Indexer() directly. Triggered by utils.processor.run_xinclude_updater(). """ log = log_helper.setup_logger() indexer = Indexer( log ) for updated_file_path in files_updated: if indexer.check_updated_file_path( updated_file_path ): q.enqueue_call( func=u'usep_gh_handler_app.utils.indexer.run_update_entry', kwargs={u'updated_file_path': updated_file_path} ) for removed_file_path in files_removed: if indexer.check_removed_file_path( removed_file_path ): q.enqueue_call( func=u'usep_gh_handler_app.utils.indexer.run_remove_entry', kwargs={u'removed_file_path': removed_file_path} ) return
def run_xinclude_updater( files_to_update, files_to_remove ): """ Updates the three <xi:include href="../path/inscription.xml"> hrefs in each inscription file. Reason is that the folder structure as exists for editors and on github is slightly different than in web-app. Triggered bu utils.processor.run_copy_files(). """ log = log_helper.setup_logger() log.debug( u'in utils.processor.run_call_xinclude_replacer(); starting' ) assert type( files_to_update ) == list; assert type( files_to_remove ) == list xinclude_updater = XIncludeUpdater( log ) xinclude_updater.update_xinclude_references() log.debug( u'in processor.run_call_xinclude_updater(); enqueuing next job' ) q.enqueue_call( func=u'usep_gh_handler_app.utils.indexer.run_update_index', kwargs={u'files_updated': files_to_update, u'files_removed': files_to_remove} ) return
def run_copy_files( files_to_update, files_to_remove ): """ Runs a copy and then triggers an index job. Incoming data not for copying, but to pass to indexer. Triggered by utils.processor.run_call_git_pull(). """ log = log_helper.setup_logger() log.debug( u'in utils.processor.run_copy_files(); starting' ) assert type( files_to_update ) == list; assert type( files_to_remove ) == list log.debug( u'in utils.processor.run_copy_files(); files_to_update, `%s`' % pprint.pformat(files_to_update) ) log.debug( u'in utils.processor.run_copy_files(); files_to_remove, `%s`' % pprint.pformat(files_to_remove) ) copier = Copier( log ) copier.copy_files() q.enqueue_call( func=u'usep_gh_handler_app.utils.processor.run_xinclude_updater', kwargs={u'files_to_update': files_to_update, u'files_to_remove': files_to_remove} ) return
def run_call_git_pull( files_to_process ): """ Initiates a git pull update. Spawns a call to Processor.process_file() for each result found. Triggered by usep_gh_handler.handle_github_push(). """ log = log_helper.setup_logger() assert sorted( files_to_process.keys() ) == [ u'files_removed', u'files_updated', u'timestamp']; log.debug( u'in utils.processor.run_call_git_pull(); files_to_process, `%s`' % pprint.pformat(files_to_process) ) time.sleep( 2 ) # let any existing jobs in process finish ( puller, copier ) = ( Puller(log), Copier(log) ) puller.call_git_pull() ( files_to_update, files_to_remove ) = ( copier.get_files_to_update(files_to_process), copier.get_files_to_remove(files_to_process) ) log.debug( u'in utils.processor.run_call_git_pull(); enqueuing next job' ) q.enqueue_call( func=u'usep_gh_handler_app.utils.processor.run_copy_files', kwargs={u'files_to_update': files_to_update, u'files_to_remove': files_to_remove} ) return
def run_start_reindex_all(): """ Starts process by building a list of inscriptions. Process: - build list of all inscriptions (from file system) that need to be indexed. - build list of all inscription_ids that need to be removed, meaning inscription_ids that _are_ in solr and are _not_ in the file system inscriptions list. - enqueue all add jobs - enqueue all remove jobs Triggered by utils.processor.run_simple_copy_files(). """ log = log_helper.setup_logger() filenames_builder = InscriptionFilenamesBuilder( log ) inscriptions = filenames_builder.build_inscription_filepaths() q.enqueue_call( func=u'usep_gh_handler_app.utils.reindex_all_support.run_build_solr_remove_list', kwargs={u'inscriptions': inscriptions} ) return
# -*- coding: utf-8 -*- import datetime, json, os, pprint, urlparse import flask, redis, requests, rq from flask.ext.basicauth import BasicAuth # http://flask-basicauth.readthedocs.org/en/latest/ from usep_gh_handler_app.utils import log_helper, reindex_all_support from usep_gh_handler_app.utils.web_app_helper import WebAppHelper ## setup B_AUTH_PASSWORD = unicode( os.environ[u'usep_gh__BASIC_AUTH_PASSWORD'] ) B_AUTH_USERNAME = unicode( os.environ[u'usep_gh__BASIC_AUTH_USERNAME'] ) app = flask.Flask(__name__) log = log_helper.setup_logger() app.config[u'BASIC_AUTH_USERNAME'] = B_AUTH_USERNAME app.config[u'BASIC_AUTH_PASSWORD'] = B_AUTH_PASSWORD basic_auth = BasicAuth(app) app_helper = WebAppHelper( log ) q = rq.Queue( u'usep', connection=redis.Redis() ) @app.route( u'/reindex_all/', methods=[u'GET'] ) @basic_auth.required def reindex_all(): """ Triggers a git-pull and a re-index of everything. Called via admin. """ try: log.debug( u'in usep_gh_handler.reindex_all(); starting' ) q.enqueue_call ( func=u'usep_gh_handler_app.utils.reindex_all_support.run_call_simple_git_pull', kwargs = {} )