def sanity_check(self):
"""Perform post-initialization sanity checks.
This is run on every startup to check that the database table schema matches our model definitions. If there are un-run migrations this will bail out and do not let the problem to escalate later.
See also: :ref:`websauna.sanity_check`.
"""
from websauna.system.model import sanitycheck
from websauna.system.model.meta import Base
from websauna.system.model.meta import create_dbsession
from websauna.system.core import redis
dbsession = create_dbsession(self.config.registry)
if not sanitycheck.is_sane_database(Base, dbsession):
raise SanityCheckFailed(
"The database sanity check failed. Check log for details.")
dbsession.close()
if self._has_redis_sessions:
if not redis.is_sane_redis(self.config):
raise SanityCheckFailed(
"Could not connect to Redis server.\nWebsauna is configured to use Redis server for session data.\nIt cannot start up without a running Redis server.\nPlease consult your operating system community how to install and start a Redis server."
)
def sanity_check(self):
"""Perform post-initialization sanity checks.
This is run on every startup to check that the database table schema matches our model definitions. If there are un-run migrations this will bail out and do not let the problem to escalate later.
See also: :ref:`websauna.sanity_check`.
"""
import sqlalchemy.exc
from websauna.system.model import sanitycheck
from websauna.system.model.meta import Base
from websauna.system.model.meta import create_dbsession
from websauna.system.core import redis
dbsession = create_dbsession(self.config.registry)
db_connection_string = self.config.registry.settings.get(
"sqlalchemy.url")
try:
if not sanitycheck.is_sane_database(Base, dbsession):
raise SanityCheckFailed(
"The database sanity check failed. Check log for details.")
except sqlalchemy.exc.OperationalError as e:
raise SanityCheckFailed(
"The database {} is not responding.\nMake sure the database is running on your local computer or correctly configured in settings INI file.\nFor more information see https://websauna.org/docs/tutorials/gettingstarted/tutorial_02.html."
.format(db_connection_string)) from e
dbsession.close()
if self._has_redis_sessions:
if not redis.is_sane_redis(self.config):
raise SanityCheckFailed(
"Could not connect to Redis server.\nWebsauna is configured to use Redis server for session data.\nIt cannot start up without a running Redis server.\nPlease consult your operating system community how to install and start a Redis server."
)
def make_routable_request(dbsession: Optional[Session],
registry: Registry,
path="/") -> IRequest:
"""Creates a dummy request that has route_url and other routing methods.
As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL.
See also :func:`make_dummy_request`.
:param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``.
"""
base_url = registry.get("websauna.site_url", None)
# TODO: Honour request_factory here
request = Request.blank(path, base_url=base_url)
# apply_request_extensions()?
request.registry = registry
request.user = None
if dbsession:
request.dbsession = dbsession
else:
tm = TransactionManager()
dbsession = create_dbsession(request.registry, tm)
request.dbsession = dbsession
request.tm = request.transaction_manager = tm
def terminate_session(request):
# Close db session at the end of the request and return the db connection back to the pool
dbsession.close()
request.add_finished_callback(terminate_session)
return request
def create_test_dbsession(request, settings: dict, transaction_manager=transaction.manager) -> Session:
"""Create a test database session and setup database.
Create and drop all tables when called. Add teardown function py.test to drop all tables during teardown.
:param request: py.test test request
:param settings: test.ini app settings
:param transaction_manager:
:return: New database session
"""
from websauna.system.model.meta import Base
dbsession = create_dbsession(settings, manager=transaction_manager)
engine = dbsession.get_bind()
with transaction.manager:
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
def teardown():
# There might be open transactions in the database. They will block DROP ALL and thus the tests would end up in a deadlock. Thus, we clean up all connections we know about.
# XXX: Fix this shit
with transaction.manager:
Base.metadata.drop_all(engine)
dbsession.close()
request.addfinalizer(teardown)
return dbsession
def init_websauna_script_env(config_uri: str) -> dict:
"""Initialize Websauna WSGI application for a IPython notebook.
:param config_uri: Path to config INI file
:return: Dictionary of shell variables
"""
options = {"sanity_check": False}
app = get_wsgi_app(config_uri, defaults=options)
initializer = initializer_from_app(app)
registry = initializer.config.registry
dbsession = create_dbsession(registry)
pyramid_env = scripting.prepare(registry=app.initializer.config.registry)
pyramid_env["app"] = app
pyramid_env["initializer"] = initializer
# Websauna specific
# Set up the request with websauna.site_url setting as the base URL
request = make_routable_request(dbsession, registry)
pyramid_env["request"] = request
pyramid_env["dbsession"] = dbsession
return pyramid_env
def create_test_dbsession(request,
settings: dict,
transaction_manager=transaction.manager) -> Session:
"""Create a test database session and setup database.
Create and drop all tables when called. Add teardown function py.test to drop all tables during teardown.
:param request: py.test test request
:param settings: test.ini app settings
:param transaction_manager:
:return: New database session
"""
from websauna.system.model.meta import Base
dbsession = create_dbsession(settings, manager=transaction_manager)
engine = dbsession.get_bind()
with transaction.manager:
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
def teardown():
# There might be open transactions in the database. They will block DROP ALL and thus the tests would end up in a deadlock. Thus, we clean up all connections we know about.
# XXX: Fix this shit
with transaction.manager:
Base.metadata.drop_all(engine)
dbsession.close()
request.addfinalizer(teardown)
return dbsession
def sanity_check(self):
"""Perform post-initialization sanity checks.
This is run on every startup to check that the database table schema matches our model definitions. If there are un-run migrations this will bail out and do not let the problem to escalate later.
See also: :ref:`websauna.sanity_check`.
"""
import sqlalchemy.exc
from websauna.system.model import sanitycheck
from websauna.system.model.meta import Base
from websauna.system.model.meta import create_dbsession
from websauna.system.core import redis
dbsession = create_dbsession(self.config.registry)
db_connection_string = self.config.registry.settings.get("sqlalchemy.url")
try:
if not sanitycheck.is_sane_database(Base, dbsession):
raise SanityCheckFailed("The database sanity check failed. Check log for details.")
except sqlalchemy.exc.OperationalError as e:
raise SanityCheckFailed("The database {} is not responding. Make sure the database is running on your local computer or correctly configured in settings INI file. For more information see https://websauna.org/docs/tutorials/gettingstarted/tutorial_02.html.".format(db_connection_stringg)) from e
dbsession.close()
if self._has_redis_sessions:
if not redis.is_sane_redis(self.config):
raise SanityCheckFailed("Could not connect to Redis server.\nWebsauna is configured to use Redis server for session data.\nIt cannot start up without a running Redis server.\nPlease consult your operating system community how to install and start a Redis server.")
def create_test_dbsession(request, registry: Registry, transaction_manager=transaction.manager) -> Session:
"""Create a test database session and setup database.
Create and drop all tables when called. Add teardown function py.test to drop all tables during teardown.
Also add implicit UUID extension on the database, so we don't need to add by hand every time.
:param request: py.test test request
:param settings: test.ini app settings
:param transaction_manager:
:return: New database session
"""
from websauna.system.model.meta import Base
dbsession = create_dbsession(registry, manager=transaction_manager)
engine = dbsession.get_bind()
connection = engine.connect()
connection.execute('create extension if not exists "uuid-ossp";')
with transaction.manager:
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
def teardown():
# There might be open transactions in the database. They will block DROP ALL and thus the tests would end up in a deadlock. Thus, we clean up all connections we know about.
# XXX: Fix this shit
with transaction.manager:
Base.metadata.drop_all(engine)
dbsession.close()
request.addfinalizer(teardown)
return dbsession
def init_websauna_script_env(config_uri: str) -> dict:
"""Initialize Websauna WSGI application for a IPython notebook.
:param config_uri: Path to config INI file
:return: Dictionary of shell variables
"""
monkey_patch_paster_config_parser()
setup_logging(config_uri)
bootstrap_env = bootstrap(config_uri, options=dict(sanity_check=False))
app = bootstrap_env["app"]
initializer = getattr(app, "initializer", None)
assert initializer is not None, "Configuration did not yield to Websauna application with Initializer set up"
registry = initializer.config.registry
dbsession = create_dbsession(registry)
pyramid_env = scripting.prepare(registry=app.initializer.config.registry)
pyramid_env["app"] = app
pyramid_env["initializer"] = initializer
# Websauna specific
# Set up the request with websauna.site_url setting as the base URL
request = make_routable_request(dbsession, registry)
pyramid_env["request"] = request
pyramid_env["dbsession"] = dbsession
return pyramid_env
def setUp(self):
# Create a threadh-local automatic session factory
self.session = create_dbsession(self.registry)
self.engine = self.session.get_bind()
# Load Bitcoin models to play around with
with transaction.manager:
Base.metadata.create_all(self.engine, tables=[TestModel.__table__, DefautDataTestModel.__table__])
def setUp(self):
# Create a threadh-local automatic session factory
self.session = create_dbsession(self.registry)
self.engine = self.session.get_bind()
# Load Bitcoin models to play around with
with transaction.manager:
Base.metadata.create_all(
self.engine,
tables=[TestModel.__table__, DefautDataTestModel.__table__])
def init_websauna(config_uri: str,
sanity_check: bool = False,
console_app: bool = False,
extra_options: dict = None) -> Request:
"""Initialize Websauna WSGI application for a command line oriented script.
Example:
.. code-block:: python
import sys
from websauna.system.devop.cmdline import init_websauna
config_uri = sys.argv[1]
request = init_websauna(config_uri)
:param config_uri: Path to config INI file
:param sanity_check: Perform database sanity check on start
:param console_app: Set true to setup console-mode logging. See :func:`setup_console_logging`
:param extra_options: Passed through bootstrap() and is available as :attr:`websauna.system.Initializer.global_options`.
:return: Faux Request object pointing to a site root, having registry and every configured.
"""
# Paster thinks we are a string
if sanity_check:
sanity_check = "true"
else:
sanity_check = "false"
options = {"sanity_check": sanity_check}
if extra_options:
options.update(extra_options)
app = get_wsgi_app(config_uri, defaults=options)
initializer = initializer_from_app(app)
registry = initializer.config.registry
dbsession = create_dbsession(registry)
# Set up the request with websauna.site_url setting as the base URL
request = make_routable_request(dbsession, registry)
# This exposes the app object for the integration tests e.g test_static_asset
# TODO: Find a cleaner way to do this
request.app = app
return request
def make_routable_request(dbsession: t.Optional[Session] = None,
registry: t.Optional[Registry] = None,
path='/') -> IRequest:
"""Creates a dummy request that has route_url and other routing methods.
As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL.
See also :func:`make_dummy_request`.
TODO: Split this to two different functions: one for existing dbsession and one for where dbsession is connected.
:param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``.
:param registry: Configuration registry
:param path: Path being requested.
:return: Current request.
"""
base_url = registry.settings.get("websauna.site_url", None)
# TODO: Honour request_factory here
request = Request.blank(path, base_url=base_url)
request.registry = registry
request.user = None
request.view_name = ''
# This will create request.tm, others
apply_request_extensions(request)
if dbsession:
# Use the provided dbsession for this request
request.dbsession = dbsession
if hasattr(dbsession, "transaction_manager"):
request.tm = request.transaction_manager = dbsession.transaction_manager
else:
# Create a new dbsession and transaction manager for this request
tm = TransactionManager()
dbsession = create_dbsession(request.registry, tm)
request.dbsession = dbsession
request.tm = request.transaction_manager = tm
def terminate_session(request):
# Close db session at the end of the request and return the db connection back to the pool
dbsession.close()
request.add_finished_callback(terminate_session)
return request
def make_routable_request(dbsession: Optional[Session]=None, registry: Registry=None, path="/") -> IRequest:
"""Creates a dummy request that has route_url and other routing methods.
As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL.
See also :func:`make_dummy_request`.
TODO: Split this to two different functions: one for existing dbsession and one for where dbsession is connected.
:param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``.
"""
base_url = registry.settings.get("websauna.site_url", None)
# TODO: Honour request_factory here
request = Request.blank(path, base_url=base_url)
# TODO: do apply_request_extensions()?
request.registry = registry
request.user = None
request.view_name = ''
# This will create request.tm, others
apply_request_extensions(request)
if dbsession:
# Use the provided dbsession for this request
request.dbsession = dbsession
if hasattr(dbsession, "transaction_manager"):
request.tm = request.transaction_manager = dbsession.transaction_manager
else:
# Create a new dbsession and transaction manager for this request
tm = TransactionManager()
dbsession = create_dbsession(request.registry, tm)
request.dbsession = dbsession
request.tm = request.transaction_manager = tm
def terminate_session(request):
# Close db session at the end of the request and return the db connection back to the pool
dbsession.close()
request.add_finished_callback(terminate_session)
return request
def setup(self, dbsession=None):
request = self.request
if not dbsession:
dbsession = create_dbsession(request.registry)
logger.info("Setting up Ethereum service %s with dbsession %s", self, dbsession)
self.web3 = self.create_web3()
with dbsession.transaction_manager:
network = get_eth_network(dbsession, self.name)
network_id = network.id
self.geth = self.start_geth()
self.service = EthereumService(self.web3, network_id, dbsession, request.registry)
self.do_unlock()
logger.info("setup() complete")
def sanity_check(self):
"""Perform post-initialization sanity checks.
This is run on every startup to check that the database table schema matches our model definitions. If there are un-run migrations this will bail out and do not let the problem to escalate later.
"""
from websauna.system.model import sanitycheck
from websauna.system.model.meta import Base
from websauna.system.model.meta import create_dbsession
from websauna.system.core import redis
dbsession = create_dbsession(self.config.registry)
if not sanitycheck.is_sane_database(Base, dbsession):
raise SanityCheckFailed("The database sanity check failed. Check log for details.")
dbsession.close()
if self._has_redis_sessions:
if not redis.is_sane_redis(self.config):
raise SanityCheckFailed("Could not connect to Redis server.\nWebsauna is configured to use Redis server for session data.\nIt cannot start up without a running Redis server.\nPlease consult your operating system community how to install and start a Redis server.")
def setup(self, dbsession=None):
request = self.request
if not dbsession:
dbsession = create_dbsession(request.registry)
logger.info("Setting up Ethereum service %s with dbsession %s", self, dbsession)
host = self.config["host"]
port = int(self.config["port"])
self.web3 = web3 = Web3(KeepAliveRPCProvider(host, port, connection_timeout=20, network_timeout=20))
with dbsession.transaction_manager:
network = get_eth_network(dbsession, self.name)
network_id = network.id
self.geth = self.start_geth()
self.service = EthereumService(web3, network_id, dbsession, request.registry)
self.do_unlock()
logger.info("setup() complete")
def create_test_dbsession(request: FixtureRequest,
registry: Registry,
transaction_manager=transaction.manager) -> Session:
"""Create a test database session and setup database.
Create and drop all tables when called. Add teardown function py.test to drop all tables during teardown.
Also add implicit UUID extension on the database, so we don't need to add by hand every time.
:param request: py.test test request
:param settings: test.ini app settings
:param transaction_manager:
:return: New database session
"""
from websauna.system.model.meta import Base
dbsession = create_dbsession(registry, manager=transaction_manager)
engine = dbsession.get_bind()
connection = engine.connect()
# Support native PSQL UUID types
if engine.dialect.name == "postgresql":
connection.execute('create extension if not exists "uuid-ossp";')
with transaction.manager:
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
def teardown():
# There might be open transactions in the database. They will block DROP ALL and thus the tests would end up in a deadlock. Thus, we clean up all connections we know about.
# XXX: Fix this shit
with transaction.manager:
Base.metadata.drop_all(engine)
dbsession.close()
request.addfinalizer(teardown)
return dbsession
def custom_dbsession(request, app: Router, transaction_manager=transaction.manager) -> Session:
from websauna.system.model.meta import Base
dbsession = create_dbsession(app.initializer.config.registry.settings, manager=transaction_manager)
engine = dbsession.get_bind()
with transaction.manager:
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
def teardown():
# There might be open transactions in the database. They will block DROP ALL and thus the tests would end up in a deadlock. Thus, we clean up all connections we know about.
# XXX: Fix this shit
with transaction.manager:
Base.metadata.drop_all(engine)
dbsession.close()
request.addfinalizer(teardown)
return dbsession
def factory():
dbsession = create_dbsession(test_request.registry, manager=None)
# Retry each transaction max 1 times
dbsession.transaction_manager.retry_attempt_count = 2
return dbsession
def factory():
dbsession = create_dbsession(test_request.registry, manager=None)
# Retry each transaction max 1 times
dbsession.transaction_manager.retry_attempt_count = 2
return dbsession
