def do_index(doc_type, search_api_url, search_api_access_token, data_api_url, data_api_access_token, mapping, serial, index, frameworks): logger.info("Search API URL: {search_api_url}", extra={'search_api_url': search_api_url}) logger.info("Data API URL: {data_api_url}", extra={'data_api_url': data_api_url}) if serial: mapper = map else: pool = ThreadPool(10) mapper = pool.imap_unordered indexer = indexers[doc_type]( doc_type, dmapiclient.DataAPIClient(data_api_url, data_api_access_token), dmapiclient.SearchAPIClient(search_api_url, search_api_access_token), index) if mapping and search_mapping_matches_framework(mapping, frameworks): indexer.create_index(mapping=mapping) counter = 0 start_time = datetime.utcnow() status = True items = indexer.request_items(frameworks) for result in mapper(indexer, items): counter += 1 status = status and result print_progress(counter, start_time) return status
from functools import wraps from flask import Flask from flask.ext.bootstrap import Bootstrap from flask.ext.sqlalchemy import SQLAlchemy import dmapiclient from dmutils import init_app, flask_featureflags from config import configs bootstrap = Bootstrap() db = SQLAlchemy() search_api_client = dmapiclient.SearchAPIClient() feature_flags = flask_featureflags.FeatureFlag() def create_app(config_name): application = Flask(__name__) application.config['DM_ENVIRONMENT'] = config_name init_app(application, configs[config_name], bootstrap=bootstrap, db=db, feature_flags=feature_flags, search_api_client=search_api_client) if not application.config['DM_API_AUTH_TOKENS']: raise Exception("No DM_API_AUTH_TOKENS provided") from .main import main as main_blueprint