def do_index(doc_type, search_api_url, search_api_access_token, data_api_url, data_api_access_token, mapping, serial,
index, frameworks):
logger.info("Search API URL: {search_api_url}", extra={'search_api_url': search_api_url})
logger.info("Data API URL: {data_api_url}", extra={'data_api_url': data_api_url})
if serial:
mapper = map
else:
pool = ThreadPool(10)
mapper = pool.imap_unordered
indexer = indexers[doc_type](
doc_type,
dmapiclient.DataAPIClient(data_api_url, data_api_access_token),
dmapiclient.SearchAPIClient(search_api_url, search_api_access_token),
index)
if mapping and search_mapping_matches_framework(mapping, frameworks):
indexer.create_index(mapping=mapping)
counter = 0
start_time = datetime.utcnow()
status = True
items = indexer.request_items(frameworks)
for result in mapper(indexer, items):
counter += 1
status = status and result
print_progress(counter, start_time)
return status
from functools import wraps
from flask import Flask
from flask.ext.bootstrap import Bootstrap
from flask.ext.sqlalchemy import SQLAlchemy
import dmapiclient
from dmutils import init_app, flask_featureflags
from config import configs
bootstrap = Bootstrap()
db = SQLAlchemy()
search_api_client = dmapiclient.SearchAPIClient()
feature_flags = flask_featureflags.FeatureFlag()
def create_app(config_name):
application = Flask(__name__)
application.config['DM_ENVIRONMENT'] = config_name
init_app(application,
configs[config_name],
bootstrap=bootstrap,
db=db,
feature_flags=feature_flags,
search_api_client=search_api_client)
if not application.config['DM_API_AUTH_TOKENS']:
raise Exception("No DM_API_AUTH_TOKENS provided")
from .main import main as main_blueprint