def connect(url=default_url, echo=False):
"""
Connect to a dusqlite database
Args:
url: SQLAlchemy compatible URL to the database
echo (bool): Echo SQL commands to stderr
Returns a database ``connection`` to be passed to other dusqlite functions
"""
engine = sa.create_engine(url, echo=echo)
# Load closure extension
def load_closure(dbapi_conn, unused):
dbapi_conn.enable_load_extension(True)
ext, _ = os.path.splitext(
pkg_resources.resource_filename(__name__, 'closure.so'))
dbapi_conn.load_extension(ext)
dbapi_conn.enable_load_extension(False)
sa.event.listen(engine, 'connect', load_closure)
connection = engine.connect()
# Apply migrations
config = alembic.config.Config()
config.set_main_option('script_location', 'dusqlite:alembic')
config.attributes['connection'] = connection
alembic.command.upgrade(config, 'head')
return connection
def init():
"""Generates a new migration"""
directory = current_app.extensions[EXT_NAME].directory
config = Config()
config.set_main_option('script_location', directory)
config.config_file_name = os.path.join(directory, 'alembic.ini')
alembic.command.init(config, directory, 'flask')
def create_alembic_env_context(metadata, func=None):
config = alembic.config.Config()
config.set_main_option('script_location', 'bkr.server:alembic')
script = alembic.script.ScriptDirectory.from_config(config)
return alembic.environment.EnvironmentContext(config=config,
script=script,
fn=func)
def init():
"""Generates a new migration"""
directory = current_app.extensions[EXT_NAME].directory
config = Config()
config.set_main_option('script_location', directory)
config.config_file_name = os.path.join(directory, 'alembic.ini')
alembic.command.init(config, directory, 'flask')
def _create_alembic_config(self) -> alembic.config.Config:
alembic_dir = os.path.join(os.path.dirname(__file__), "alembic")
config = alembic.config.Config(os.path.join(os.path.dirname(__file__), "alembic.ini"))
config.set_main_option("script_location", escape_alembic_config_value(alembic_dir))
config.set_main_option("sqlalchemy.url", escape_alembic_config_value(self.url))
return config
def apply_migrations():
"""Applies all alembic migrations."""
alembic_config = os.path.join(os.path.dirname(__file__), "../",
"alembic.ini")
config = alembic.config.Config(alembic_config)
app_config = make_config()
config.set_main_option("sqlalchemy.url", app_config["DATABASE_URI"])
alembic.command.upgrade(config, "head")
def alembic_migration(migrations_path: str,
alembic_init_path: str = 'alembic.ini',
connection=None,
revision: str = 'head'):
config = alembic.config.Config(alembic_init_path)
config.set_main_option('script_location', migrations_path)
if connection is not None:
config.attributes['connection'] = connection
alembic.command.upgrade(config, revision)
def get_alembic_config(config_path, echo):
"""Create and return an alembic config."""
config = alembic.config.Config(config_path)
# Set logging based on the --sql flag
config.set_main_option('asb_do_logging', str(echo))
return config
def get_alembic_config(config_path, echo):
"""Create and return an alembic config."""
config = alembic.config.Config(config_path)
# Set logging based on the --sql flag
config.set_main_option('asb_do_logging', str(echo))
return config
def reverse_all(DATABASE_URL: str) -> None:
import_models()
engine = create_engine(DATABASE_URL, echo=False)
connection: Connection = engine.connect()
config = alembic.config.Config(str(ALEMBIC_INI))
config.set_main_option("script_location", str(MIGRATIONS_DIR))
config.set_main_option("sqlalchemy.url", DATABASE_URL)
config.attributes["connection"] = connection # pylint: disable=E1137
alembic.command.downgrade(config, "base")
def _create_alembic_config(self):
# type: () -> alembic.config.Config
alembic_dir = os.path.join(os.path.dirname(__file__), 'alembic')
config = alembic.config.Config(os.path.join(os.path.dirname(__file__), 'alembic.ini'))
config.set_main_option('script_location', escape_alembic_config_value(alembic_dir))
config.set_main_option('sqlalchemy.url', escape_alembic_config_value(self.url))
return config
def check_if_version_is_active(self, p_version):
from alembic.script import ScriptDirectory
from alembic.config import Config
config = Config()
config.set_main_option("script_location", "little_brother:alembic")
script = ScriptDirectory.from_config(config)
revision = script.get_revision(p_version)
return revision is not None
def _get_config(x_arg=None):
directory = current_app.extensions[EXT_NAME].directory
config = Config(os.path.join(directory, 'alembic.ini'))
config.set_main_option('script_location', directory)
if x_arg is not None:
if config.cmd_opts is None:
config.cmd_opts = lambda: None
if not getattr(config.cmd_opts, 'x', None):
config.cmd_opts.x = []
config.cmd_opts.x.append(x_arg)
return config
def _get_config(x_arg=None):
directory = current_app.extensions[EXT_NAME].directory
config = Config(os.path.join(directory, 'alembic.ini'))
config.set_main_option('script_location', directory)
if x_arg is not None:
if config.cmd_opts is None:
config.cmd_opts = lambda: None
if not getattr(config.cmd_opts, 'x', None):
config.cmd_opts.x = []
config.cmd_opts.x.append(x_arg)
return config
def database(argv):
import alembic.config
click.secho('Database: ' + app.config['SQLALCHEMY_DATABASE_URI'], fg='yellow')
config = alembic.config.Config(file_=os.path.join(_here(), 'migrations/alembic.ini'))
config.set_main_option('script_location', os.path.join(_here(), 'migrations'))
config.set_main_option('sqlalchemy.url', app.config['SQLALCHEMY_DATABASE_URI'])
cmd = alembic.config.CommandLine('inv database')
cmd.run_cmd(config, cmd.parser.parse_args(argv))
def create_all(DATABASE_URL: str) -> None:
import_models()
engine = create_engine(DATABASE_URL, echo=False)
if not database_exists(engine.url): # pragma: no cover
create_database(engine.url)
connection: Connection = engine.connect()
config = alembic.config.Config(str(ALEMBIC_INI))
config.set_main_option("script_location", str(MIGRATIONS_DIR))
config.set_main_option("sqlalchemy.url", DATABASE_URL)
config.attributes["connection"] = connection # pylint: disable=E1137
alembic.command.upgrade(config, "head")
def build_alembic_config(temboard_config):
config = alembic.config.Config()
config.set_main_option(
'sqlalchemy.url',
format_dsn(temboard_config.repository),
)
config.set_main_option(
'script_location',
os.path.dirname(__file__) + '/alembic',
)
return config
def get_alembic_config(db_url: str = DEFAULT_DB,
cmd_opts: dict = None) -> alembic.config.Config:
"""Generate the alembic configuration from the template and populate fields.
db_url: str [default: 'sqlite:////tmp/ngshare.db']
The database url used to populate sqlalchemy.url, e.g. `sqlite:///ngshare.db`.
"""
config = alembic.config.Config(ALEMBIC_INI_TEMPLATE_PATH,
cmd_opts=cmd_opts)
config.set_main_option("script_location", ALEMBIC_DIR)
config.set_main_option("sqlalchemy.url", db_url)
return config
def get_app_config(self, app, db):
"""Return alembic config for an app."""
# TODO: read these from django db settings
version_table = (getattr(app, "version_table", None)
or db.kwargs.get("version_table")
or "alembic_version_%s" %
app.label.lower().replace(".", "_"))
max_length = db.engine.dialect.max_identifier_length
if max_length and len(version_table) >= max_length:
raise CommandError(
"'{name}' is {length} characters long which is an invalid identifier "
"in {dialect!r} as its max idenfier length is {max_length}".
format(name=version_table,
dialect=db.engine.dialect.name,
length=len(version_table),
max_length=max_length))
version_table_schema = getattr(
app, "version_table_schema",
None) or db.kwargs.get("version_table_schema")
config = alembic.config.Config(output_buffer=self.stdout,
stdout=self.stdout)
config.set_main_option("script_location",
SORCERY_ALEMBIC_CONFIG_FOLDER)
config.set_main_option("version_locations",
self.get_app_version_path(app))
config.set_main_option("version_table", version_table)
if version_table_schema and db.engine.dialect.name != "sqlite":
config.set_main_option("version_table_schema",
version_table_schema)
return config
def run_alembic_operation(metadata, func):
# We intentionally *don't* run inside the normal Alembic env.py so that we
# can force the use of the SA metadata we are given, rather than using the
# normal global TurboGears metadata instance. Ultimately this is to make
# the migration testable.
config = alembic.config.Config()
config.set_main_option('script_location', 'bkr.server:alembic')
script = alembic.script.ScriptDirectory.from_config(config)
env_context = alembic.environment.EnvironmentContext(config=config,
script=script, fn=func)
connection = metadata.bind.connect()
env_context.configure(connection=connection, target_metadata=metadata)
env_context.run_migrations()
def _migrate(self):
"""Perform the alembic migrations for this connection"""
with self.engine.begin() as conn:
context = alembic.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
self.log.debug('Current migration revision: %s' % current_rev)
config = alembic.config.Config()
config.set_main_option("script_location",
"zuul:alembic/sql_reporter")
config.set_main_option("sqlalchemy.url",
self.connection_config.get('dburi'))
alembic.command.upgrade(config, 'head')
def run_alembic_operation(metadata, func):
# We intentionally *don't* run inside the normal Alembic env.py so that we
# can force the use of the SA metadata we are given, rather than using the
# normal global TurboGears metadata instance. Ultimately this is to make
# the migration testable.
config = alembic.config.Config()
config.set_main_option('script_location', 'bkr.server:alembic')
script = alembic.script.ScriptDirectory.from_config(config)
env_context = alembic.environment.EnvironmentContext(config=config,
script=script,
fn=func)
connection = metadata.bind.connect()
env_context.configure(connection=connection, target_metadata=metadata)
env_context.run_migrations()
def migrate(self):
conn = self.engine.connect()
context = alembic.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
self.log.debug('Current migration revision: %s' % current_rev)
has_table = self.engine.dialect.has_table(conn, "project")
config = alembic.config.Config()
config.set_main_option("script_location", "gertty:alembic")
config.set_main_option("sqlalchemy.url", self.app.config.dburi)
if current_rev is None and has_table:
self.log.debug('Stamping database as initial revision')
alembic.command.stamp(config, "44402069e137")
alembic.command.upgrade(config, 'head')
def migrate(self):
conn = self.engine.connect()
context = alembic.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
self.log.debug('Current migration revision: %s' % current_rev)
has_table = self.engine.dialect.has_table(conn, "project")
config = alembic.config.Config()
config.set_main_option("script_location", "gertty:alembic")
config.set_main_option("sqlalchemy.url", self.app.config.dburi)
if current_rev is None and has_table:
self.log.debug('Stamping database as initial revision')
alembic.command.stamp(config, "44402069e137")
alembic.command.upgrade(config, 'head')
def migrate(self, app):
conn = self.engine.connect()
context = alembic.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
self.log.debug('Current migration revision: %s' % current_rev)
has_table = self.engine.dialect.has_table(conn, "repository")
config = alembic.config.Config()
config.set_main_option("script_location", "hubtty:alembic")
config.set_main_option("sqlalchemy.url", self.dburi)
config.hubtty_app = app
if current_rev is None and has_table:
self.log.debug('Stamping database as initial revision')
alembic.command.stamp(config, "a2af1e2e44ee")
alembic.command.upgrade(config, 'head')
def _migrate(self):
"""Perform the alembic migrations for this connection"""
with self.engine.begin() as conn:
context = alembic.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
self.log.debug('Current migration revision: %s' % current_rev)
config = alembic.config.Config()
config.set_main_option("script_location",
"zuul:driver/sql/alembic")
config.set_main_option("sqlalchemy.url",
self.connection_config.get('dburi'))
# Alembic lets us add arbitrary data in the tag argument. We can
# leverage that to tell the upgrade scripts about the table prefix.
tag = {'table_prefix': self.table_prefix}
alembic.command.upgrade(config, 'head', tag=tag)
def _migrate(self):
"""Perform the alembic migrations for this connection"""
with self.engine.begin() as conn:
context = alembic.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
self.log.debug('Current migration revision: %s' % current_rev)
config = alembic.config.Config()
config.set_main_option("script_location",
"zuul:driver/sql/alembic")
config.set_main_option("sqlalchemy.url",
self.connection_config.get('dburi'))
# Alembic lets us add arbitrary data in the tag argument. We can
# leverage that to tell the upgrade scripts about the table prefix.
tag = {'table_prefix': self.table_prefix}
alembic.command.upgrade(config, 'head', tag=tag)
def make_migration(args_):
target_db_setup(args_.target_db, args_.target_db_user, args_.target_db_password, args_.target_db_host,
args_.target_db_port, args_.db_name)
if not schema_verification():
sys.exit()
tables_ = create_tables_from_rules(True)
config = alembic.config.Config("./alembic.ini")
config.set_main_option("sqlalchemy.url", target_db)
config.set_main_option("script_location", "./migrate")
script = ScriptDirectory.from_config(config)
head_revision = ""
try:
command.revision(config, autogenerate=True)
head_revision = script.get_current_head()
migrate()
except FileNotFoundError as Error:
print("Error Occurred")
print(Error)
print("Rollback Migration")
try:
file = os.path.join(os.curdir, os.path.join("migrate/versions", (head_revision + "_.py")))
os.remove(file)
except FileNotFoundError:
pass
finally:
sys.exit()
except CommandError as E:
print(E)
print("Target Database not in sync")
print("Re-Run migration")
try:
engine.execute("drop table alembic_version")
purge("./migrate/versions", "\w+.py")
except exc.ProgrammingError:
pass
finally:
pass
return tables_
def set_migration(self):
db_folder = Path(__file__).parent.parent / 'db'
config = alembic.config.Config()
config.set_main_option('script_location', str(db_folder / 'alembic'))
config.set_main_option('version_locations',
str(db_folder / 'alembic/versions'))
config.set_main_option('sqlalchemy.url',
f'sqlite:///{str(self.sqlite_db)}')
alembic.command.upgrade(config, 'head')
def upgrade(uri):
current_dir = os.path.dirname(__file__)
config = alembic.config.Config(f'{current_dir}/alembic.ini')
config.set_main_option('script_location', f'{current_dir}/alembic')
config.set_main_option('sqlalchemy.url', uri)
config.set_main_option('configure_logging', 'false')
logger.info('Upgrading database')
engine = create_engine(uri)
wait_is_ready(engine)
alembic.command.upgrade(config, 'head')
logger.info('Database upgraded')
def createdb():
"""Create a new empty database with a single administrator."""
print("* Creating database schema")
# Create the database schema
db.create_all()
print("* Adding alembic stamp")
# Create alembic_version table
migrations_directory = current_app.extensions['migrate'].directory
config = alembic.config.Config(
os.path.join(migrations_directory, 'alembic.ini'))
config.set_main_option('script_location', migrations_directory)
alembic.command.stamp(config, "head")
# Add required groups
print("* Adding administrators' and 'BC' groups")
_add_group('administrators')
_add_group('BC')
# Add educations, which must be present to create the administrator user
print("* Adding educations")
education_names = [
"BSc Informatica",
"BSc Kunstmatige Intelligentie",
"BSc Informatiekunde",
"MSc Information Studies",
"MSc Software Engineering",
"MSc System and Network Engineering",
"MSc Artificial Intelligence",
"MSc Logic",
"MSc Computational Science",
"MSc Computer Science",
"MSc Medical Informatics",
"MSc Grid Computing",
"Other",
"Minor programmeren",
"Minor Informatica",
"Minor Kunstmatige Intelligentie"]
for name in education_names:
if not Education.query.filter(Education.name == name).first():
db.session.add(Education(name=name))
else:
print("-> Education {} exists".format(name))
db.session.commit()
# Add some default navigation
print("* Adding default navigation entries")
navigation_entries = [
('via', 'via', '/via', False, [
('Nieuws', 'News', '/news/', False, []),
('PimPy', 'PimPy', '/pimpy', False, []),
('Commissies', 'Committees', '/commissie', False, []),
('Admin', 'Admin', '/admin', False, [
('Navigatie', 'Navigation', '/navigation', False, []),
('Formulieren', 'Forms', '/forms', False, []),
('Redirect', 'Redirect', '/redirect', False, []),
('Users', 'Users', '/users', False, []),
('Groups', 'Groups', '/groups', False, []),
('Files', 'Files', '/files', False, [])
]),
]),
('Activiteiten', 'Activities', '/activities', True, [
('Activiteiten Archief', 'Activities archive',
'/activities/archive', False, []),
('Activiteiten Overzicht', 'Activities overview',
'/activities/view', False, [])
]),
('Vacatures', 'Vacancies', '/vacancies/', False, []),
('Tentamenbank', 'Examinations', '/examination', False, []),
('Samenvattingen', 'Summaries', '/summary', False, [])
]
_add_navigation(navigation_entries)
print("* Adding administrator user")
first_name = prompt("\tFirst name")
last_name = prompt("\tLast name")
email_regex = re.compile("^[^@]+@[^@]+\.[^@]+$")
while True:
email = prompt("\tEmail")
if email_regex.match(email):
break
print("\tInvalid email address: " + email)
while True:
passwd_plain = prompt_pass("\tPassword")
passwd_plain_rep = prompt_pass("\tRepeat password")
if passwd_plain == passwd_plain_rep:
break
print("\tPasswords do not match")
passwd = bcrypt.hashpw(passwd_plain.encode('utf-8'), bcrypt.gensalt())
admin = User(
first_name=first_name,
last_name=last_name,
email=email,
password=passwd,
education_id=Education.query.first().id)
admin.has_paid = True
_add_user(admin, "A user with email '{}' already exists".format(email))
# Add admin user to administrators group
admin_group = Group.query.filter_by(name='administrators').first()
admin_group.add_user(admin)
db.session.commit()
roles = []
for role in Roles:
group_role = GroupRole()
group_role.group_id = admin_group.id
group_role.role = role.name
roles.append(group_role)
# Grant read/write privilege to administrators group on every module
db.session.bulk_save_objects(roles)
db.session.commit()
print("* Adding default settings")
settings = {'SECRET_KEY': 'localsecret',
"CSRF_ENABLED": "True",
"CSRF_SESSION_KEY": "localsession",
"RECAPTCHA_PUBLIC_KEY": "",
"RECAPTCHA_PRIVATE_KEY": "",
"GOOGLE_SERVICE_EMAIL": "*****@*****.**",
"GOOGLE_CALENDAR_ID": "",
"ELECTIONS_NOMINATE_START": "2014-12-12",
"ELECTIONS_VOTE_START": "2015-01-05",
"ELECTIONS_VOTE_END": "2015-01-16",
"GITLAB_TOKEN": "",
"MOLLIE_URL": "https://api.mollie.nl/v1/payments/",
"MOLLIE_KEY": "",
"COPERNICA_ENABLED": "False",
"COPERNICA_API_KEY": "",
"COPERNICA_DATABASE_ID": "",
"COPERNICA_ACTIEPUNTEN": "",
"COPERNICA_ACTIVITEITEN": "",
"COPERNICA_NEWSLETTER_TOKEN": "",
"DOMJUDGE_ADMIN_USERNAME": "******",
"DOMJUDGE_ADMIN_PASSWORD": "",
"DOMJUDGE_URL": "",
"DOMJUDGE_USER_PASSWORD": "",
"SENTRY_DSN": "DUMMY",
"ENVIRONMENT": "Development",
"PRIVACY_POLICY_URL_EN": "/static/via_privacy_policy_nl.pdf",
"PRIVACY_POLICY_URL_NL": "/static/via_privacy_policy_en.pdf"}
for key, value in settings.items():
if Setting.query.filter(Setting.key == key).count() > 1:
print(f"-> {key} already exists")
else:
db.session.add(Setting(key=key, value=value))
print(f"-> {key} added to database.")
db.session.commit()
print("Done!")
def get_alembic_config(self):
config = alembic.config.Config()
config.set_main_option('script_location', self.db_migrations_path)
config.set_main_option('sqlalchemy.url', str(self.engine.url))
config.set_main_option('url', str(self.engine.url))
return config
def reverse_all(connection: Connection, db_url: str) -> None:
config = alembic.config.Config(str(ALEMBIC_INI))
config.set_main_option("script_location", str(VERSIONS_DIR))
config.set_main_option("sqlalchemy.url", db_url)
cast(Any, config.attributes)["connection"] = connection
alembic.command.downgrade(config, "base")
#!/usr/bin/env python3
import sys
from os.path import dirname, realpath
from pathlib import Path
import alembic # type: ignore
import alembic.command # type: ignore
import alembic.config # type: ignore
SRC_ROOT = Path(dirname(realpath(__file__))).parent
ASSETS_DIR = SRC_ROOT / "src" / "spellbot" / "assets"
ALEMBIC_INI = ASSETS_DIR / "alembic.ini"
VERSIONS_DIR = SRC_ROOT / "src" / "spellbot" / "versions"
url = sys.argv[1]
message = sys.argv[2]
config = alembic.config.Config(str(ALEMBIC_INI))
config.set_main_option("script_location", str(VERSIONS_DIR))
config.set_main_option("sqlalchemy.url", url)
alembic.command.revision(config, message=message, autogenerate=True)
def createdb():
"""Create a new empty database with a single administrator."""
print("* Creating database schema")
# Create the database schema
db.create_all()
print("* Adding alembic stamp")
# Create alembic_version table
migrations_directory = current_app.extensions['migrate'].directory
config = alembic.config.Config(
os.path.join(migrations_directory, 'alembic.ini'))
config.set_main_option('script_location', migrations_directory)
alembic.command.stamp(config, "head")
# Add required groups
print("* Adding 'all','administrators' and 'BC' groups")
_add_group('all')
_add_group('administrators')
_add_group('BC')
# Add educations, which must be present to create the administrator user
print("* Adding educations")
education_names = [
"BSc Informatica",
"BSc Kunstmatige Intelligentie",
"BSc Informatiekunde",
"MSc Information Studies",
"MSc Software Engineering",
"MSc System and Network Engineering",
"MSc Artificial Intelligence",
"MSc Logic",
"MSc Computational Science",
"MSc Computer Science",
"MSc Medical Informatics",
"MSc Grid Computing",
"Other",
"Minor programmeren",
"Minor Informatica",
"Minor Kunstmatige Intelligentie"]
db.session.bulk_save_objects(Education(name) for name in education_names)
db.session.commit()
# Add some default navigation
print("* Adding default navigation entries")
navigation_entries = [
('via', 'via', '/via', False, [
('Nieuws', 'News', '/news/', False, []),
('PimPy', 'PimPy', '/pimpy', False, []),
('Commissies', 'Committees', '/commissie', False, []),
('Admin', 'Admin', '/admin', False, [
('Navigatie', 'Navigation', '/navigation', False, []),
('Formulieren', 'Forms', '/forms', False, []),
('Redirect', 'Redirect', '/redirect', False, []),
('Users', 'Users', '/users', False, []),
('Groups', 'Groups', '/groups', False, []),
('Files', 'Files', '/files', False, [])
]),
]),
('Activiteiten', 'Activities', '/activities', True, [
('Activiteiten Archief', 'Activities archive',
'/activities/archive', False, []),
('Activiteiten Overzicht', 'Activities overview',
'/activities/view', False, [])
]),
('Vacatures', 'Vacancies', '/vacancies/', False, []),
('Tentamenbank', 'Examinations', '/examination', False, []),
('Samenvattingen', 'Summaries', '/summary', False, [])
]
_add_navigation(navigation_entries)
print("* Adding administrator user")
first_name = prompt("\tFirst name")
last_name = prompt("\tLast name")
email_regex = re.compile("^[^@]+@[^@]+\.[^@]+$")
while True:
email = prompt("\tEmail")
if email_regex.match(email):
break
print("\tInvalid email address: " + email)
while True:
passwd_plain = prompt_pass("\tPassword")
passwd_plain_rep = prompt_pass("\tRepeat password")
if passwd_plain == passwd_plain_rep:
break
print("\tPasswords do not match")
passwd = bcrypt.hashpw(passwd_plain, bcrypt.gensalt())
admin = User(
first_name=first_name,
last_name=last_name,
email=email,
password=passwd,
education_id=Education.query.first().id)
admin.has_paid = True
_add_user(admin, True,
"A user with email '{}' already exists".format(email))
# Add admin user to administrators group
admin_group = Group.query.filter_by(name='administrators').first()
admin_group.add_user(admin)
db.session.commit()
# Grant read/write privilege to administrators group on every module
db.session.bulk_save_objects(
GroupPermission(module, admin_group.id, 2) for module in
app.blueprints.keys())
db.session.commit()
print("Done!")
def init_database(connection_url):
"""
Create and initialize the database engine. This must be done before the
session object can be used. This will also attempt to perform any updates to
the database schema if the backend support such operations.
:param str connection_url: The url for the database connection.
:return: The initialized database engine.
"""
connection_url = normalize_connection_url(connection_url)
connection_url = sqlalchemy.engine.url.make_url(connection_url)
logger.info("initializing database connection with driver {0}".format(
connection_url.drivername))
if connection_url.drivername == 'sqlite':
engine = sqlalchemy.create_engine(
connection_url,
connect_args={'check_same_thread': False},
poolclass=sqlalchemy.pool.StaticPool)
sqlalchemy.event.listens_for(
engine, 'begin')(lambda conn: conn.execute('BEGIN'))
elif connection_url.drivername == 'postgresql':
engine = sqlalchemy.create_engine(connection_url)
else:
raise errors.KingPhisherDatabaseError(
'only sqlite and postgresql database drivers are supported')
Session.remove()
Session.configure(bind=engine)
try:
models.Base.metadata.create_all(engine)
except sqlalchemy.exc.SQLAlchemyError as error:
error_lines = map(lambda line: line.strip(), error.message.split('\n'))
raise errors.KingPhisherDatabaseError('SQLAlchemyError: ' +
' '.join(error_lines).strip())
session = Session()
set_meta_data('database_driver',
connection_url.drivername,
session=session)
schema_version = (get_meta_data('schema_version', session=session)
or models.SCHEMA_VERSION)
session.commit()
session.close()
logger.debug("current database schema version: {0} ({1}current)".format(
schema_version,
('' if schema_version == models.SCHEMA_VERSION else 'not ')))
if schema_version > models.SCHEMA_VERSION:
raise errors.KingPhisherDatabaseError(
'the database schema is for a newer version, automatic downgrades are not supported'
)
elif schema_version < models.SCHEMA_VERSION:
alembic_config_file = find.find_data_file('alembic.ini')
if not alembic_config_file:
raise errors.KingPhisherDatabaseError(
'cannot find the alembic.ini configuration file')
alembic_directory = find.find_data_directory('alembic')
if not alembic_directory:
raise errors.KingPhisherDatabaseError(
'cannot find the alembic data directory')
config = alembic.config.Config(alembic_config_file)
config.config_file_name = alembic_config_file
config.set_main_option('script_location', alembic_directory)
config.set_main_option('skip_logger_config', 'True')
config.set_main_option('sqlalchemy.url', str(connection_url))
logger.warning(
"automatically updating the database schema to version {0}".format(
models.SCHEMA_VERSION))
try:
alembic.command.upgrade(config, 'head')
except Exception as error:
logger.critical(
"database schema upgrade failed with exception: {0}.{1} {2}".
format(error.__class__.__module__, error.__class__.__name__,
getattr(error, 'message', '')).rstrip())
raise errors.KingPhisherDatabaseError(
'failed to upgrade to the latest database schema')
set_meta_data('schema_version', models.SCHEMA_VERSION)
logger.debug("connected to {0} database: {1}".format(
connection_url.drivername, connection_url.database))
return engine
def init_database(connection_url, extra_init=False):
"""
Create and initialize the database engine. This must be done before the
session object can be used. This will also attempt to perform any updates to
the database schema if the backend supports such operations.
:param str connection_url: The url for the database connection.
:param bool extra_init: Run optional extra dbms-specific initialization logic.
:return: The initialized database engine.
"""
connection_url = normalize_connection_url(connection_url)
connection_url = sqlalchemy.engine.url.make_url(connection_url)
logger.info("initializing database connection with driver {0}".format(
connection_url.drivername))
if connection_url.drivername == 'sqlite':
engine = sqlalchemy.create_engine(
connection_url,
connect_args={'check_same_thread': False},
poolclass=sqlalchemy.pool.StaticPool)
sqlalchemy.event.listens_for(
engine, 'begin')(lambda conn: conn.execute('BEGIN'))
elif connection_url.drivername == 'postgresql':
if extra_init:
init_database_postgresql(connection_url)
engine = sqlalchemy.create_engine(
connection_url, connect_args={'client_encoding': 'utf8'})
else:
raise errors.KingPhisherDatabaseError(
'only sqlite and postgresql database drivers are supported')
Session.remove()
Session.configure(bind=engine)
inspector = sqlalchemy.inspect(engine)
if 'campaigns' not in inspector.get_table_names():
logger.debug('campaigns table not found, creating all new tables')
try:
models.Base.metadata.create_all(engine)
except sqlalchemy.exc.SQLAlchemyError as error:
error_lines = (line.strip() for line in error.message.split('\n'))
raise errors.KingPhisherDatabaseError(
'SQLAlchemyError: ' + ' '.join(error_lines).strip())
schema_version = get_schema_version(engine)
logger.debug("current database schema version: {0} ({1})".format(
schema_version,
('latest' if schema_version == models.SCHEMA_VERSION else 'obsolete')))
if 'alembic_version' not in inspector.get_table_names():
logger.debug(
'alembic version table not found, attempting to create and set version'
)
init_alembic(engine, schema_version)
if schema_version > models.SCHEMA_VERSION:
raise errors.KingPhisherDatabaseError(
'the database schema is for a newer version, automatic downgrades are not supported'
)
elif schema_version < models.SCHEMA_VERSION:
alembic_config_file = find.data_file('alembic.ini')
if not alembic_config_file:
raise errors.KingPhisherDatabaseError(
'cannot find the alembic.ini configuration file')
alembic_directory = find.data_directory('alembic')
if not alembic_directory:
raise errors.KingPhisherDatabaseError(
'cannot find the alembic data directory')
config = alembic.config.Config(alembic_config_file)
config.config_file_name = alembic_config_file
config.set_main_option('script_location', alembic_directory)
config.set_main_option('skip_logger_config', 'True')
config.set_main_option('sqlalchemy.url', str(connection_url))
logger.warning(
"automatically updating the database schema from version {0} to {1}"
.format(schema_version, models.SCHEMA_VERSION))
try:
alembic.command.upgrade(config, 'head')
except Exception as error:
logger.critical(
"database schema upgrade failed with exception: {0}.{1} {2}".
format(error.__class__.__module__, error.__class__.__name__,
getattr(error, 'message', '')).rstrip(),
exc_info=True)
raise errors.KingPhisherDatabaseError(
'failed to upgrade to the latest database schema')
logger.info(
"successfully updated the database schema from version {0} to {1}".
format(schema_version, models.SCHEMA_VERSION))
# reset it because it may have been altered by alembic
Session.remove()
Session.configure(bind=engine)
set_metadata('database_driver', connection_url.drivername)
set_metadata('last_started', datetime.datetime.utcnow())
set_metadata('schema_version', models.SCHEMA_VERSION)
logger.debug("connected to {0} database: {1}".format(
connection_url.drivername, connection_url.database))
signals.db_initialized.send(connection_url)
return engine
def init_database(connection_url, extra_init=False):
"""
Create and initialize the database engine. This must be done before the
session object can be used. This will also attempt to perform any updates to
the database schema if the backend supports such operations.
:param str connection_url: The url for the database connection.
:param bool extra_init: Run optional extra dbms-specific initialization logic.
:return: The initialized database engine.
"""
connection_url = normalize_connection_url(connection_url)
connection_url = sqlalchemy.engine.url.make_url(connection_url)
logger.info("initializing database connection with driver {0}".format(connection_url.drivername))
if connection_url.drivername == 'sqlite':
engine = sqlalchemy.create_engine(connection_url, connect_args={'check_same_thread': False}, poolclass=sqlalchemy.pool.StaticPool)
sqlalchemy.event.listens_for(engine, 'begin')(lambda conn: conn.execute('BEGIN'))
elif connection_url.drivername == 'postgresql':
if extra_init:
init_database_postgresql(connection_url)
engine = sqlalchemy.create_engine(connection_url)
else:
raise errors.KingPhisherDatabaseError('only sqlite and postgresql database drivers are supported')
Session.remove()
Session.configure(bind=engine)
inspector = sqlalchemy.inspect(engine)
if not 'meta_data' in inspector.get_table_names():
logger.debug('meta_data table not found, creating all new tables')
try:
models.Base.metadata.create_all(engine)
except sqlalchemy.exc.SQLAlchemyError as error:
error_lines = (line.strip() for line in error.message.split('\n'))
raise errors.KingPhisherDatabaseError('SQLAlchemyError: ' + ' '.join(error_lines).strip())
session = Session()
set_meta_data('database_driver', connection_url.drivername, session=session)
schema_version = (get_meta_data('schema_version', session=session) or models.SCHEMA_VERSION)
session.commit()
session.close()
logger.debug("current database schema version: {0} ({1})".format(schema_version, ('latest' if schema_version == models.SCHEMA_VERSION else 'obsolete')))
if not 'alembic_version' in inspector.get_table_names():
logger.debug('alembic version table not found, attempting to create and set version')
init_alembic(engine, schema_version)
if schema_version > models.SCHEMA_VERSION:
raise errors.KingPhisherDatabaseError('the database schema is for a newer version, automatic downgrades are not supported')
elif schema_version < models.SCHEMA_VERSION:
alembic_config_file = find.data_file('alembic.ini')
if not alembic_config_file:
raise errors.KingPhisherDatabaseError('cannot find the alembic.ini configuration file')
alembic_directory = find.data_directory('alembic')
if not alembic_directory:
raise errors.KingPhisherDatabaseError('cannot find the alembic data directory')
config = alembic.config.Config(alembic_config_file)
config.config_file_name = alembic_config_file
config.set_main_option('script_location', alembic_directory)
config.set_main_option('skip_logger_config', 'True')
config.set_main_option('sqlalchemy.url', str(connection_url))
logger.warning("automatically updating the database schema from version {0} to {1}".format(schema_version, models.SCHEMA_VERSION))
try:
alembic.command.upgrade(config, 'head')
except Exception as error:
logger.critical("database schema upgrade failed with exception: {0}.{1} {2}".format(error.__class__.__module__, error.__class__.__name__, getattr(error, 'message', '')).rstrip(), exc_info=True)
raise errors.KingPhisherDatabaseError('failed to upgrade to the latest database schema')
logger.info("successfully updated the database schema from version {0} to {1}".format(schema_version, models.SCHEMA_VERSION))
# reset it because it may have been altered by alembic
Session.remove()
Session.configure(bind=engine)
session = Session()
set_meta_data('schema_version', models.SCHEMA_VERSION)
logger.debug("connected to {0} database: {1}".format(connection_url.drivername, connection_url.database))
signals.db_initialized.send(connection_url)
return engine
def run(self, args) -> int:
if (hasattr(args, 'cert') is True and hasattr(args, 'key') is False) or \
(hasattr(args, 'cert') is False and hasattr(args, 'key') is True):
self.logger.error("Both the cert and key is required to use SSL")
return 1
elif hasattr(args, 'cert') is True and hasattr(args, 'key') is True:
cherrypy.server.ssl_certificate = args.cert.name
cherrypy.server.ssl_private_key = args.key.name
s3_parameters = {}
if hasattr(args, 's3_endpoint'):
s3_parameters['endpoint_url'] = args.s3_endpoint
if hasattr(args, 's3_access_key_id'):
s3_parameters['aws_access_key_id'] = args.s3_access_key_id
if hasattr(args, 's3_secret_access_key'):
s3_parameters['aws_secret_access_key'] = args.s3_secret_access_key
self.logger.info("Connecting to S3")
s3_client = boto3.resource('s3', **s3_parameters).meta.client
try:
s3_client.head_bucket(Bucket=args.s3_bucket)
except botocore.exceptions.ClientError as e:
self.logger.error("Error checking if bucket exists: %s", e)
return 1
except botocore.exceptions.NoCredentialsError as e:
self.logger.error("Error loading S3 credentials: %s", e)
return 1
self.logger.info("Connecting to database")
database = Database(db_url=args.db_url, pool_size=args.db_pool_size)
database.connect()
self.logger.info("Running SQL migrations")
config = alembic.config.Config("alembic.ini")
config.set_main_option("sqlalchemy.url", database.db_url)
config.attributes['connection'] = database.engine.connect()
alembic.command.upgrade(config, 'head')
http_app = Application(logging_config=None, debug=True)
http_app.register_mount(RootMount(http_app, database, args.s3_bucket, s3_client))
http_app.setup()
self.logger.info("Running CherryPy Webserver")
cherrypy.config.update({
'global': {
'environment': 'production',
'server.socket_host': str(args.bind_address),
'server.socket_port': args.port,
}
})
cherrypy.engine.start()
cherrypy.engine.block()
database.engine.dispose()
return 0
def create_alembic_env_context(metadata, func=None):
config = alembic.config.Config()
config.set_main_option('script_location', 'bkr.server:alembic')
script = alembic.script.ScriptDirectory.from_config(config)
return alembic.environment.EnvironmentContext(config=config,
script=script, fn=func)
def migrate():
config = alembic.config.Config("./alembic.ini")
config.set_main_option("sqlalchemy.url", target_db)
config.set_main_option("script_location", "./migrate")
command.upgrade(config, "head")
