def create_web_app() -> Flask:
"""Initialize and configure the accounts application."""
app = Flask('registry')
app.config.from_pyfile('config.py')
# app.register_blueprint(ui.blueprint)
datastore.init_app(app)
SessionStore.init_app(app)
Base(app) # Gives us access to the base UI templates and resources.
auth.Auth(app) # Handless sessions and authn/z.
oauth2.init_app(app)
app.register_blueprint(blueprint)
middleware = [AuthMiddleware]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
app.jinja_env.filters['scope_label'] = filters.scope_label
if app.config['CREATE_DB']:
with app.app_context():
datastore.create_all()
register_error_handlers(app)
return app
def test_token_is_usable(self, mock_get_config):
"""Verify that :func:`.helpers.generate_token` makes usable tokens."""
mock_get_config.return_value = {'JWT_SECRET': 'thesecret'}
os.environ['JWT_SECRET'] = 'thesecret'
scope = [auth.scopes.VIEW_SUBMISSION, auth.scopes.EDIT_SUBMISSION,
auth.scopes.CREATE_SUBMISSION]
token = helpers.generate_token("1234", "*****@*****.**", "theuser",
scope=scope)
app = Flask('test')
app.config['JWT_SECRET'] = 'thesecret'
Base(app)
auth.Auth(app) # <- Install the Auth extension.
wrap(app, [auth.middleware.AuthMiddleware]) # <- Install middleware.
@app.route('/')
@auth.decorators.scoped(auth.scopes.EDIT_SUBMISSION)
def protected():
return "this is protected"
client = app.test_client()
with app.app_context():
response = client.get('/')
self.assertEqual(response.status_code,
status.HTTP_401_UNAUTHORIZED)
response = client.get('/', headers={'Authorization': token})
self.assertEqual(response.status_code, status.HTTP_200_OK)
def create_web_app() -> Flask:
"""Initialize and configure the accounts application."""
app = Flask('accounts')
app.config.from_pyfile('config.py')
SessionStore.init_app(app)
legacy.init_app(app)
users.init_app(app)
app.register_blueprint(ui.blueprint)
Base(app) # Gives us access to the base UI templates and resources.
auth.Auth(app) # Handless sessions and authn/z.
s3.init_app(app)
middleware = [auth.middleware.AuthMiddleware]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
if app.config['CREATE_DB']:
with app.app_context():
legacy.create_all()
users.create_all()
return app
def create_classic_api_web_app() -> Flask:
"""Initialize an instance of the search frontend UI web application."""
logging.getLogger("boto").setLevel(logging.ERROR)
logging.getLogger("boto3").setLevel(logging.ERROR)
logging.getLogger("botocore").setLevel(logging.ERROR)
app = Flask("search")
app.json_encoder = ISO8601JSONEncoder
app.config.from_pyfile("config.py") # type: ignore
index.SearchSession.init_app(app)
Base(app)
auth.Auth(app)
app.register_blueprint(classic_api.blueprint)
wrap(
app,
[request_logs.ClassicLogsMiddleware, auth.middleware.AuthMiddleware],
)
for error, handler in classic_api.exceptions.get_handlers():
app.errorhandler(error)(handler)
return app
def create_ui_web_app() -> Flask:
"""Initialize an instance of the search frontend UI web application."""
logging.getLogger('boto').setLevel(logging.ERROR)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)
app = Flask('search')
app.config.from_pyfile('config.py') # type: ignore
app.url_map.converters['archive'] = ArchiveConverter
index.SearchSession.init_app(app)
Base(app)
app.register_blueprint(ui.blueprint)
s3.init_app(app)
wrap(app, [request_logs.ClassicLogsMiddleware])
# app.config['PROFILE'] = True
# app.config['DEBUG'] = True
# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[100], sort_by=('cumtime', ))
for filter_name, template_filter in filters.filters:
app.template_filter(filter_name)(template_filter)
return app
def setUp(self):
"""Instantiate an app and attach middlware."""
self.secret = 'foosecret'
self.app = Flask('foo')
os.environ['JWT_SECRET'] = self.secret
self.app.register_blueprint(blueprint)
wrap(self.app, [AuthMiddleware])
self.client = self.app.test_client()
def test_uwsgi_is_not_available(self):
"""Attach base middleware when uwsgi is not available."""
if 'uwsgi' in sys.modules:
del sys.modules['uwsgi']
from arxiv.base.middleware import request_logs, wrap
app = Flask('test')
wrap(app, [request_logs.ClassicLogsMiddleware])
self.assertIsInstance(app.wsgi_app, request_logs.BaseMiddleware,
"BaseMiddleware is attached instead")
def create_api_app() -> Flask:
"""Create a new API application."""
app = Flask('funding')
app.config.from_pyfile('config.py')
Base(app)
auth.Auth(app)
app.register_blueprint(routes.api.blueprint)
wrap(app, [auth.middleware.AuthMiddleware])
register_error_handlers(app)
return app
def _create_base_app() -> Flask:
app = Flask('zero')
app.config.from_pyfile('config.py')
app.json_encoder = ISO8601JSONEncoder
baz.BazService.init_app(app)
things.init_app(app)
Base(app) # Gives us access to the base UI templates and resources.
auth.Auth(app) # Sets up authn/z machinery.
wrap(app, [auth.middleware.AuthMiddleware])
return app
def test_uwsgi_is_available(self):
"""Attach logging middleware to app when uwsgi is available."""
mock_uwsgi = mock.MagicMock()
sys.modules['uwsgi'] = mock_uwsgi
from arxiv.base.middleware import request_logs, wrap
app = Flask('test')
wrap(app, [request_logs.ClassicLogsMiddleware])
self.assertIsInstance(app.wsgi_app, request_logs.ClassicLogsMiddleware,
"ClassicLogsMiddleware should be the outermost"
" middleware")
app(self.environ, mock.MagicMock())
self.assertGreater(mock_uwsgi.set_logvar.call_count, 0,
"Should set logging variables for uwsgi")
def create_web_app() -> Flask:
"""Initialize and configure the accounts application."""
app = Flask('accounts')
app.config.from_pyfile('config.py')
sessions.init_app(app)
legacy.init_app(app)
users.init_app(app)
app.register_blueprint(ui.blueprint)
Base(app) # Gives us access to the base UI templates and resources.
auth.Auth(app) # Handless sessions and authn/z.
wrap(app, [auth.middleware.AuthMiddleware])
return app
def create_web_app() -> Flask:
"""Initialize an instance of the extractor backend service."""
app = Flask('metadata')
classic.init_app(app)
app.config.from_pyfile('config.py')
app.register_blueprint(routes.blueprint)
app.errorhandler(Forbidden)(jsonify_exception)
app.errorhandler(NotFound)(jsonify_exception)
app.errorhandler(BadRequest)(jsonify_exception)
app.errorhandler(Unauthorized)(jsonify_exception)
wrap(app, [auth.AuthMiddleware])
return app
def create_app() -> Flask:
app = Flask(__name__)
app.config.from_pyfile('config.py')
app.json_encoder = EnumJSONEncoder
database.init_app(app)
jira.init_app(app)
app.register_blueprint(api)
app.logger.setLevel(app.config['LOGLEVEL'])
register_error_handlers(app)
if app.config['VAULT_ENABLED']:
wrap(app, [vault.middleware.VaultMiddleware])
app.middlewares['VaultMiddleware'].update_secrets({})
return app
def create_app() -> Flask:
"""Create an instance of the compiler service app."""
from . import celeryconfig
app = Flask(__name__)
filemanager.FileManager.init_app(app)
store.Store.init_app(app)
app.config.from_pyfile('config.py')
celery_app.config_from_object(celeryconfig)
Base(app)
auth.Auth(app)
app.register_blueprint(routes.blueprint)
register_error_handlers(app)
middleware = [auth.middleware.AuthMiddleware]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
# Leaving this here for future performance tuning. - Erick
#
# app.config['PROFILE'] = True
# app.config['DEBUG'] = True
# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[100],
# sort_by=('cumtime', ))
#
if app.config['WAIT_FOR_SERVICES']:
with app.app_context(): # type: ignore
logger.info('initialize and wait for upstream services')
# Adding a wait here can help keep boto3 from getting stuck if
# we are starting localstack at the same time. This can probably
# just be 0 (default) in production.
time.sleep(app.config['WAIT_ON_STARTUP'])
filemanager_service = filemanager.FileManager.current_session()
store_service = store.Store.current_session()
store_service.initialize()
wait_for(filemanager_service)
if app.config['WAIT_FOR_WORKER']:
wait_for(compiler, await_result=True) # type: ignore
logger.info('All upstream services are available; ready to start')
return app
def create_app() -> Flask:
"""Initialize an instance of the authenticator service."""
app = Flask('authenticator')
app.config.from_pyfile('config.py')
Base(app)
SessionStore.init_app(app)
if app.config['VAULT_ENABLED']:
wrap(app, [vault.middleware.VaultMiddleware])
app.middlewares['VaultMiddleware'].update_secrets({})
app.register_blueprint(routes.blueprint)
app.errorhandler(NotFound)(jsonify_exception)
app.errorhandler(BadRequest)(jsonify_exception)
app.errorhandler(Unauthorized)(jsonify_exception)
app.errorhandler(Forbidden)(jsonify_exception)
return app
def create_app() -> Flask:
"""Create a new agent application."""
app = Flask(__name__)
app.config.from_object(config)
app.config.add_hook('SUBMISSION_AGENT_DATABASE_URI', update_binds)
Base(app)
# Register logging and secrets middleware.
middleware = [request_logs.ClassicLogsMiddleware]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
# Make sure that we have all of the secrets that we need to run.
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
# Initialize services.
database.init_app(app)
mail.init_app(app)
Classifier.init_app(app)
Compiler.init_app(app)
PlainTextService.init_app(app)
init_app(app)
if app.config['WAIT_FOR_SERVICES']:
time.sleep(app.config['WAIT_ON_STARTUP'])
with app.app_context():
wait_for(database)
wait_for(Classifier.current_session(),
timeout=app.config['CLASSIFIER_STATUS_TIMEOUT'])
wait_for(Compiler.current_session(),
timeout=app.config['COMPILER_STATUS_TIMEOUT'])
wait_for(PlainTextService.current_session(),
timeout=app.config['PLAINTEXT_STATUS_TIMEOUT'])
# FILEMANAGER_STATUS_TIMEOUT
logger.info('All upstream services are available; ready to start')
with app.app_context():
if not database.tables_exist():
database.create_all()
return app
def create_ui_web_app() -> Flask:
"""Initialize an instance of the search frontend UI web application."""
logging.getLogger('boto').setLevel(logging.ERROR)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)
app = Flask('search')
app.config.from_pyfile('config.py')
index.init_app(app)
Base(app)
app.register_blueprint(ui.blueprint)
s3.init_app(app)
wrap(app, [request_logs.ClassicLogsMiddleware])
return app
def setUp(self, mock_SecretsManager):
"""We have a flask app."""
self.manager = mock.MagicMock()
mock_SecretsManager.return_value = self.manager
self.app = Flask(__name__)
self.app.config['VAULT_HOST'] = 'foohost'
self.app.config['VAULT_PORT'] = '1234'
self.app.config['VAULT_CERT'] = '/path/to/cert'
self.app.config['VAULT_REQUESTS'] = [{
'type': 'generic',
'name': 'JWT_SECRET',
'mount_point': 'wherethesecretslive',
'path': 'jwt',
'key': 'secret'
}]
self.app.config['VAULT_ROLE'] = 'foovaultrole'
self.app.config['KUBE_TOKEN'] = 'fookubetoken1234'
wrap(self.app, [middleware.VaultMiddleware])
self.client = self.app.test_client()
def create_web_app() -> Flask:
"""Initialize and configure the accounts application."""
app = Flask('registry')
app.config.from_pyfile('config.py')
# app.register_blueprint(ui.blueprint)
datastore.init_app(app)
sessions.init_app(app)
Base(app) # Gives us access to the base UI templates and resources.
auth.Auth(app) # Handless sessions and authn/z.
oauth2.init_app(app)
app.register_blueprint(blueprint)
wrap(app, [auth.middleware.AuthMiddleware])
app.jinja_env.filters['scope_label'] = filters.scope_label
datastore.create_all()
return app
def create_web_app(for_worker: bool = False) -> Flask:
"""Initialize an instance of the web application."""
app = Flask('fulltext')
app.config.from_pyfile('config.py')
app.url_map.converters['source'] = SubmissionSourceConverter
if app.config['LOGLEVEL'] < 40:
# Make sure that boto doesn't spam the logs when we're in debug mode.
pylogging.getLogger('boto').setLevel(pylogging.ERROR)
pylogging.getLogger('boto3').setLevel(pylogging.ERROR)
pylogging.getLogger('botocore').setLevel(pylogging.ERROR)
Base(app)
Auth(app)
app.register_blueprint(routes.blueprint)
store.Storage.current_session().init_app(app)
legacy.CanonicalPDF.init_app(app)
preview.PreviewService.init_app(app)
middleware = [auth.middleware.AuthMiddleware]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
if app.config['WAIT_FOR_SERVICES']:
time.sleep(app.config['WAIT_ON_STARTUP'])
with app.app_context():
wait_for(store.Storage.current_session())
wait_for(legacy.CanonicalPDF.current_session())
wait_for(preview.PreviewService.current_session())
if for_worker:
wait_for(extractor.do_extraction)
else:
wait_for(extract, await_result=True) # type: ignore
logger.info('All upstream services are available; ready to start')
register_error_handlers(app)
app.celery_app = extract.get_or_create_worker_app(app)
return app
def create_ui_web_app() -> Flask:
"""Initialize an instance of the search frontend UI web application."""
app = Flask('submit', static_folder='static', template_folder='templates')
app.url_map.strict_slashes = False
app.config.from_pyfile('config.py')
Base(app)
auth.Auth(app)
app.register_blueprint(ui.ui)
middleware = [
request_logs.ClassicLogsMiddleware, auth.middleware.AuthMiddleware
]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
# Make sure that we have all of the secrets that we need to run.
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
for filter_name, filter_func in filters.get_filters():
app.jinja_env.filters[filter_name] = filter_func
# Initialize services.
init_app(app)
Compiler.init_app(app)
FileManager.init_app(app)
if app.config['WAIT_FOR_SERVICES']:
time.sleep(app.config['WAIT_ON_STARTUP'])
with app.app_context():
wait_for(FileManager.current_session(),
timeout=app.config['FILEMANAGER_STATUS_TIMEOUT'])
wait_for(Compiler.current_session(),
timeout=app.config['COMPILER_STATUS_TIMEOUT'])
logger.info('All upstream services are available; ready to start')
return app
def test_wrap_is_inside_out(self):
"""Order of middleware determines call order upon request."""
from arxiv.base import middleware
class FirstMiddleware(middleware.base.BaseMiddleware):
def before(self, environ, start_response):
environ['call_order'].append('first')
return environ, start_response
class SecondMiddleware(middleware.base.BaseMiddleware):
def before(self, environ, start_response):
environ['call_order'].append('second')
return environ, start_response
app = Flask('test')
self.environ['call_order'] = []
middleware.wrap(app, [FirstMiddleware, SecondMiddleware])
app(self.environ, mock.MagicMock())
self.assertEqual(self.environ['call_order'][0], 'first',
"The first middleware should be called first")
self.assertEqual(self.environ['call_order'][1], 'second',
"The second middleware should be called second")
def create_web_app() -> Flask:
"""Initialize and configure the filemanager application."""
app = Flask('filemanager')
app.config.from_pyfile('config.py')
app.json_encoder = ISO8601JSONEncoder
# Initialize file management app
uploads.init_app(app)
Base(app) # Gives us access to the base UI templates and resources.
auth.Auth(app)
app.register_blueprint(upload_api.blueprint)
middleware = [auth.middleware.AuthMiddleware]
if app.config['VAULT_ENABLED']:
middleware.insert(0, vault.middleware.VaultMiddleware)
wrap(app, middleware)
if app.config['VAULT_ENABLED']:
app.middlewares['VaultMiddleware'].update_secrets({})
register_error_handlers(app)
return app
def create_api_web_app() -> Flask:
"""Initialize an instance of the search frontend UI web application."""
logging.getLogger('boto').setLevel(logging.ERROR)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)
app = Flask('search')
app.json_encoder = ISO8601JSONEncoder
app.config.from_pyfile('config.py')
index.init_app(app)
Base(app)
auth.Auth(app)
app.register_blueprint(api.blueprint)
wrap(app,
[request_logs.ClassicLogsMiddleware, auth.middleware.AuthMiddleware])
for error, handler in api.exceptions.get_handlers():
app.errorhandler(error)(handler)
return app
def test_token_is_usable(self):
"""Verify that :func:`.helpers.generate_token` makes usable tokens."""
os.environ['JWT_SECRET'] = 'thesecret'
scope = [
auth.scopes.VIEW_SUBMISSION, auth.scopes.EDIT_SUBMISSION,
auth.scopes.CREATE_SUBMISSION
]
token = helpers.generate_token("1234",
"*****@*****.**",
"theuser",
scope=scope)
app = Flask('test')
legacy.init_app(app)
app.config.update({
'JWT_SECRET': 'thesecret',
'SQLALCHEMY_TRACK_MODIFICATIONS': False,
'SQLALCHEMY_DATABASE_URI': 'sqlite:///'
})
Base(app)
auth.Auth(app) # <- Install the Auth extension.
wrap(app, [auth.middleware.AuthMiddleware]) # <- Install middleware.
@app.route('/')
@auth.decorators.scoped(auth.scopes.EDIT_SUBMISSION)
def protected():
return "this is protected"
client = app.test_client()
with app.app_context():
response = client.get('/')
self.assertEqual(response.status_code,
status.HTTP_401_UNAUTHORIZED)
response = client.get('/', headers={'Authorization': token})
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_wrap_attaches_middleware(self):
""":func:`.middleware.wrap` attaches WSGI middleware to a Flask app."""
from arxiv.base import middleware
class FooMiddleware(object):
def __init__(self, app):
self.app = app
self.called = False
def __call__(self, environ, start_response):
self.called = True
return self.app(environ, start_response)
@property
def wsgi_app(self):
return self
app = Flask('test')
middleware.wrap(app, [FooMiddleware])
self.assertIsInstance(app.wsgi_app, FooMiddleware,
"Middleware should be attached to Flask app")
app(self.environ, mock.MagicMock())
self.assertTrue(app.wsgi_app.called,
"Middleware should be called when Flask app is called")
def test_init(self):
"""The middlware is instantiated."""
self.app = Flask(__name__)
with self.assertRaises(KeyError):
wrap(self.app, [middleware.VaultMiddleware])
