def connect(url=default_url, echo=False): """ Connect to a dusqlite database Args: url: SQLAlchemy compatible URL to the database echo (bool): Echo SQL commands to stderr Returns a database ``connection`` to be passed to other dusqlite functions """ engine = sa.create_engine(url, echo=echo) # Load closure extension def load_closure(dbapi_conn, unused): dbapi_conn.enable_load_extension(True) ext, _ = os.path.splitext( pkg_resources.resource_filename(__name__, 'closure.so')) dbapi_conn.load_extension(ext) dbapi_conn.enable_load_extension(False) sa.event.listen(engine, 'connect', load_closure) connection = engine.connect() # Apply migrations config = alembic.config.Config() config.set_main_option('script_location', 'dusqlite:alembic') config.attributes['connection'] = connection alembic.command.upgrade(config, 'head') return connection
def init(): """Generates a new migration""" directory = current_app.extensions[EXT_NAME].directory config = Config() config.set_main_option('script_location', directory) config.config_file_name = os.path.join(directory, 'alembic.ini') alembic.command.init(config, directory, 'flask')
def create_alembic_env_context(metadata, func=None): config = alembic.config.Config() config.set_main_option('script_location', 'bkr.server:alembic') script = alembic.script.ScriptDirectory.from_config(config) return alembic.environment.EnvironmentContext(config=config, script=script, fn=func)
def _create_alembic_config(self) -> alembic.config.Config: alembic_dir = os.path.join(os.path.dirname(__file__), "alembic") config = alembic.config.Config(os.path.join(os.path.dirname(__file__), "alembic.ini")) config.set_main_option("script_location", escape_alembic_config_value(alembic_dir)) config.set_main_option("sqlalchemy.url", escape_alembic_config_value(self.url)) return config
def apply_migrations(): """Applies all alembic migrations.""" alembic_config = os.path.join(os.path.dirname(__file__), "../", "alembic.ini") config = alembic.config.Config(alembic_config) app_config = make_config() config.set_main_option("sqlalchemy.url", app_config["DATABASE_URI"]) alembic.command.upgrade(config, "head")
def alembic_migration(migrations_path: str, alembic_init_path: str = 'alembic.ini', connection=None, revision: str = 'head'): config = alembic.config.Config(alembic_init_path) config.set_main_option('script_location', migrations_path) if connection is not None: config.attributes['connection'] = connection alembic.command.upgrade(config, revision)
def get_alembic_config(config_path, echo): """Create and return an alembic config.""" config = alembic.config.Config(config_path) # Set logging based on the --sql flag config.set_main_option('asb_do_logging', str(echo)) return config
def reverse_all(DATABASE_URL: str) -> None: import_models() engine = create_engine(DATABASE_URL, echo=False) connection: Connection = engine.connect() config = alembic.config.Config(str(ALEMBIC_INI)) config.set_main_option("script_location", str(MIGRATIONS_DIR)) config.set_main_option("sqlalchemy.url", DATABASE_URL) config.attributes["connection"] = connection # pylint: disable=E1137 alembic.command.downgrade(config, "base")
def _create_alembic_config(self): # type: () -> alembic.config.Config alembic_dir = os.path.join(os.path.dirname(__file__), 'alembic') config = alembic.config.Config(os.path.join(os.path.dirname(__file__), 'alembic.ini')) config.set_main_option('script_location', escape_alembic_config_value(alembic_dir)) config.set_main_option('sqlalchemy.url', escape_alembic_config_value(self.url)) return config
def check_if_version_is_active(self, p_version): from alembic.script import ScriptDirectory from alembic.config import Config config = Config() config.set_main_option("script_location", "little_brother:alembic") script = ScriptDirectory.from_config(config) revision = script.get_revision(p_version) return revision is not None
def _get_config(x_arg=None): directory = current_app.extensions[EXT_NAME].directory config = Config(os.path.join(directory, 'alembic.ini')) config.set_main_option('script_location', directory) if x_arg is not None: if config.cmd_opts is None: config.cmd_opts = lambda: None if not getattr(config.cmd_opts, 'x', None): config.cmd_opts.x = [] config.cmd_opts.x.append(x_arg) return config
def database(argv): import alembic.config click.secho('Database: ' + app.config['SQLALCHEMY_DATABASE_URI'], fg='yellow') config = alembic.config.Config(file_=os.path.join(_here(), 'migrations/alembic.ini')) config.set_main_option('script_location', os.path.join(_here(), 'migrations')) config.set_main_option('sqlalchemy.url', app.config['SQLALCHEMY_DATABASE_URI']) cmd = alembic.config.CommandLine('inv database') cmd.run_cmd(config, cmd.parser.parse_args(argv))
def create_all(DATABASE_URL: str) -> None: import_models() engine = create_engine(DATABASE_URL, echo=False) if not database_exists(engine.url): # pragma: no cover create_database(engine.url) connection: Connection = engine.connect() config = alembic.config.Config(str(ALEMBIC_INI)) config.set_main_option("script_location", str(MIGRATIONS_DIR)) config.set_main_option("sqlalchemy.url", DATABASE_URL) config.attributes["connection"] = connection # pylint: disable=E1137 alembic.command.upgrade(config, "head")
def build_alembic_config(temboard_config): config = alembic.config.Config() config.set_main_option( 'sqlalchemy.url', format_dsn(temboard_config.repository), ) config.set_main_option( 'script_location', os.path.dirname(__file__) + '/alembic', ) return config
def get_alembic_config(db_url: str = DEFAULT_DB, cmd_opts: dict = None) -> alembic.config.Config: """Generate the alembic configuration from the template and populate fields. db_url: str [default: 'sqlite:////tmp/ngshare.db'] The database url used to populate sqlalchemy.url, e.g. `sqlite:///ngshare.db`. """ config = alembic.config.Config(ALEMBIC_INI_TEMPLATE_PATH, cmd_opts=cmd_opts) config.set_main_option("script_location", ALEMBIC_DIR) config.set_main_option("sqlalchemy.url", db_url) return config
def get_app_config(self, app, db): """Return alembic config for an app.""" # TODO: read these from django db settings version_table = (getattr(app, "version_table", None) or db.kwargs.get("version_table") or "alembic_version_%s" % app.label.lower().replace(".", "_")) max_length = db.engine.dialect.max_identifier_length if max_length and len(version_table) >= max_length: raise CommandError( "'{name}' is {length} characters long which is an invalid identifier " "in {dialect!r} as its max idenfier length is {max_length}". format(name=version_table, dialect=db.engine.dialect.name, length=len(version_table), max_length=max_length)) version_table_schema = getattr( app, "version_table_schema", None) or db.kwargs.get("version_table_schema") config = alembic.config.Config(output_buffer=self.stdout, stdout=self.stdout) config.set_main_option("script_location", SORCERY_ALEMBIC_CONFIG_FOLDER) config.set_main_option("version_locations", self.get_app_version_path(app)) config.set_main_option("version_table", version_table) if version_table_schema and db.engine.dialect.name != "sqlite": config.set_main_option("version_table_schema", version_table_schema) return config
def run_alembic_operation(metadata, func): # We intentionally *don't* run inside the normal Alembic env.py so that we # can force the use of the SA metadata we are given, rather than using the # normal global TurboGears metadata instance. Ultimately this is to make # the migration testable. config = alembic.config.Config() config.set_main_option('script_location', 'bkr.server:alembic') script = alembic.script.ScriptDirectory.from_config(config) env_context = alembic.environment.EnvironmentContext(config=config, script=script, fn=func) connection = metadata.bind.connect() env_context.configure(connection=connection, target_metadata=metadata) env_context.run_migrations()
def _migrate(self): """Perform the alembic migrations for this connection""" with self.engine.begin() as conn: context = alembic.migration.MigrationContext.configure(conn) current_rev = context.get_current_revision() self.log.debug('Current migration revision: %s' % current_rev) config = alembic.config.Config() config.set_main_option("script_location", "zuul:alembic/sql_reporter") config.set_main_option("sqlalchemy.url", self.connection_config.get('dburi')) alembic.command.upgrade(config, 'head')
def migrate(self): conn = self.engine.connect() context = alembic.migration.MigrationContext.configure(conn) current_rev = context.get_current_revision() self.log.debug('Current migration revision: %s' % current_rev) has_table = self.engine.dialect.has_table(conn, "project") config = alembic.config.Config() config.set_main_option("script_location", "gertty:alembic") config.set_main_option("sqlalchemy.url", self.app.config.dburi) if current_rev is None and has_table: self.log.debug('Stamping database as initial revision') alembic.command.stamp(config, "44402069e137") alembic.command.upgrade(config, 'head')
def migrate(self, app): conn = self.engine.connect() context = alembic.migration.MigrationContext.configure(conn) current_rev = context.get_current_revision() self.log.debug('Current migration revision: %s' % current_rev) has_table = self.engine.dialect.has_table(conn, "repository") config = alembic.config.Config() config.set_main_option("script_location", "hubtty:alembic") config.set_main_option("sqlalchemy.url", self.dburi) config.hubtty_app = app if current_rev is None and has_table: self.log.debug('Stamping database as initial revision') alembic.command.stamp(config, "a2af1e2e44ee") alembic.command.upgrade(config, 'head')
def _migrate(self): """Perform the alembic migrations for this connection""" with self.engine.begin() as conn: context = alembic.migration.MigrationContext.configure(conn) current_rev = context.get_current_revision() self.log.debug('Current migration revision: %s' % current_rev) config = alembic.config.Config() config.set_main_option("script_location", "zuul:driver/sql/alembic") config.set_main_option("sqlalchemy.url", self.connection_config.get('dburi')) # Alembic lets us add arbitrary data in the tag argument. We can # leverage that to tell the upgrade scripts about the table prefix. tag = {'table_prefix': self.table_prefix} alembic.command.upgrade(config, 'head', tag=tag)
def make_migration(args_): target_db_setup(args_.target_db, args_.target_db_user, args_.target_db_password, args_.target_db_host, args_.target_db_port, args_.db_name) if not schema_verification(): sys.exit() tables_ = create_tables_from_rules(True) config = alembic.config.Config("./alembic.ini") config.set_main_option("sqlalchemy.url", target_db) config.set_main_option("script_location", "./migrate") script = ScriptDirectory.from_config(config) head_revision = "" try: command.revision(config, autogenerate=True) head_revision = script.get_current_head() migrate() except FileNotFoundError as Error: print("Error Occurred") print(Error) print("Rollback Migration") try: file = os.path.join(os.curdir, os.path.join("migrate/versions", (head_revision + "_.py"))) os.remove(file) except FileNotFoundError: pass finally: sys.exit() except CommandError as E: print(E) print("Target Database not in sync") print("Re-Run migration") try: engine.execute("drop table alembic_version") purge("./migrate/versions", "\w+.py") except exc.ProgrammingError: pass finally: pass return tables_
def set_migration(self): db_folder = Path(__file__).parent.parent / 'db' config = alembic.config.Config() config.set_main_option('script_location', str(db_folder / 'alembic')) config.set_main_option('version_locations', str(db_folder / 'alembic/versions')) config.set_main_option('sqlalchemy.url', f'sqlite:///{str(self.sqlite_db)}') alembic.command.upgrade(config, 'head')
def upgrade(uri): current_dir = os.path.dirname(__file__) config = alembic.config.Config(f'{current_dir}/alembic.ini') config.set_main_option('script_location', f'{current_dir}/alembic') config.set_main_option('sqlalchemy.url', uri) config.set_main_option('configure_logging', 'false') logger.info('Upgrading database') engine = create_engine(uri) wait_is_ready(engine) alembic.command.upgrade(config, 'head') logger.info('Database upgraded')
def createdb(): """Create a new empty database with a single administrator.""" print("* Creating database schema") # Create the database schema db.create_all() print("* Adding alembic stamp") # Create alembic_version table migrations_directory = current_app.extensions['migrate'].directory config = alembic.config.Config( os.path.join(migrations_directory, 'alembic.ini')) config.set_main_option('script_location', migrations_directory) alembic.command.stamp(config, "head") # Add required groups print("* Adding administrators' and 'BC' groups") _add_group('administrators') _add_group('BC') # Add educations, which must be present to create the administrator user print("* Adding educations") education_names = [ "BSc Informatica", "BSc Kunstmatige Intelligentie", "BSc Informatiekunde", "MSc Information Studies", "MSc Software Engineering", "MSc System and Network Engineering", "MSc Artificial Intelligence", "MSc Logic", "MSc Computational Science", "MSc Computer Science", "MSc Medical Informatics", "MSc Grid Computing", "Other", "Minor programmeren", "Minor Informatica", "Minor Kunstmatige Intelligentie"] for name in education_names: if not Education.query.filter(Education.name == name).first(): db.session.add(Education(name=name)) else: print("-> Education {} exists".format(name)) db.session.commit() # Add some default navigation print("* Adding default navigation entries") navigation_entries = [ ('via', 'via', '/via', False, [ ('Nieuws', 'News', '/news/', False, []), ('PimPy', 'PimPy', '/pimpy', False, []), ('Commissies', 'Committees', '/commissie', False, []), ('Admin', 'Admin', '/admin', False, [ ('Navigatie', 'Navigation', '/navigation', False, []), ('Formulieren', 'Forms', '/forms', False, []), ('Redirect', 'Redirect', '/redirect', False, []), ('Users', 'Users', '/users', False, []), ('Groups', 'Groups', '/groups', False, []), ('Files', 'Files', '/files', False, []) ]), ]), ('Activiteiten', 'Activities', '/activities', True, [ ('Activiteiten Archief', 'Activities archive', '/activities/archive', False, []), ('Activiteiten Overzicht', 'Activities overview', '/activities/view', False, []) ]), ('Vacatures', 'Vacancies', '/vacancies/', False, []), ('Tentamenbank', 'Examinations', '/examination', False, []), ('Samenvattingen', 'Summaries', '/summary', False, []) ] _add_navigation(navigation_entries) print("* Adding administrator user") first_name = prompt("\tFirst name") last_name = prompt("\tLast name") email_regex = re.compile("^[^@]+@[^@]+\.[^@]+$") while True: email = prompt("\tEmail") if email_regex.match(email): break print("\tInvalid email address: " + email) while True: passwd_plain = prompt_pass("\tPassword") passwd_plain_rep = prompt_pass("\tRepeat password") if passwd_plain == passwd_plain_rep: break print("\tPasswords do not match") passwd = bcrypt.hashpw(passwd_plain.encode('utf-8'), bcrypt.gensalt()) admin = User( first_name=first_name, last_name=last_name, email=email, password=passwd, education_id=Education.query.first().id) admin.has_paid = True _add_user(admin, "A user with email '{}' already exists".format(email)) # Add admin user to administrators group admin_group = Group.query.filter_by(name='administrators').first() admin_group.add_user(admin) db.session.commit() roles = [] for role in Roles: group_role = GroupRole() group_role.group_id = admin_group.id group_role.role = role.name roles.append(group_role) # Grant read/write privilege to administrators group on every module db.session.bulk_save_objects(roles) db.session.commit() print("* Adding default settings") settings = {'SECRET_KEY': 'localsecret', "CSRF_ENABLED": "True", "CSRF_SESSION_KEY": "localsession", "RECAPTCHA_PUBLIC_KEY": "", "RECAPTCHA_PRIVATE_KEY": "", "GOOGLE_SERVICE_EMAIL": "*****@*****.**", "GOOGLE_CALENDAR_ID": "", "ELECTIONS_NOMINATE_START": "2014-12-12", "ELECTIONS_VOTE_START": "2015-01-05", "ELECTIONS_VOTE_END": "2015-01-16", "GITLAB_TOKEN": "", "MOLLIE_URL": "https://api.mollie.nl/v1/payments/", "MOLLIE_KEY": "", "COPERNICA_ENABLED": "False", "COPERNICA_API_KEY": "", "COPERNICA_DATABASE_ID": "", "COPERNICA_ACTIEPUNTEN": "", "COPERNICA_ACTIVITEITEN": "", "COPERNICA_NEWSLETTER_TOKEN": "", "DOMJUDGE_ADMIN_USERNAME": "******", "DOMJUDGE_ADMIN_PASSWORD": "", "DOMJUDGE_URL": "", "DOMJUDGE_USER_PASSWORD": "", "SENTRY_DSN": "DUMMY", "ENVIRONMENT": "Development", "PRIVACY_POLICY_URL_EN": "/static/via_privacy_policy_nl.pdf", "PRIVACY_POLICY_URL_NL": "/static/via_privacy_policy_en.pdf"} for key, value in settings.items(): if Setting.query.filter(Setting.key == key).count() > 1: print(f"-> {key} already exists") else: db.session.add(Setting(key=key, value=value)) print(f"-> {key} added to database.") db.session.commit() print("Done!")
def get_alembic_config(self): config = alembic.config.Config() config.set_main_option('script_location', self.db_migrations_path) config.set_main_option('sqlalchemy.url', str(self.engine.url)) config.set_main_option('url', str(self.engine.url)) return config
def reverse_all(connection: Connection, db_url: str) -> None: config = alembic.config.Config(str(ALEMBIC_INI)) config.set_main_option("script_location", str(VERSIONS_DIR)) config.set_main_option("sqlalchemy.url", db_url) cast(Any, config.attributes)["connection"] = connection alembic.command.downgrade(config, "base")
#!/usr/bin/env python3 import sys from os.path import dirname, realpath from pathlib import Path import alembic # type: ignore import alembic.command # type: ignore import alembic.config # type: ignore SRC_ROOT = Path(dirname(realpath(__file__))).parent ASSETS_DIR = SRC_ROOT / "src" / "spellbot" / "assets" ALEMBIC_INI = ASSETS_DIR / "alembic.ini" VERSIONS_DIR = SRC_ROOT / "src" / "spellbot" / "versions" url = sys.argv[1] message = sys.argv[2] config = alembic.config.Config(str(ALEMBIC_INI)) config.set_main_option("script_location", str(VERSIONS_DIR)) config.set_main_option("sqlalchemy.url", url) alembic.command.revision(config, message=message, autogenerate=True)
def createdb(): """Create a new empty database with a single administrator.""" print("* Creating database schema") # Create the database schema db.create_all() print("* Adding alembic stamp") # Create alembic_version table migrations_directory = current_app.extensions['migrate'].directory config = alembic.config.Config( os.path.join(migrations_directory, 'alembic.ini')) config.set_main_option('script_location', migrations_directory) alembic.command.stamp(config, "head") # Add required groups print("* Adding 'all','administrators' and 'BC' groups") _add_group('all') _add_group('administrators') _add_group('BC') # Add educations, which must be present to create the administrator user print("* Adding educations") education_names = [ "BSc Informatica", "BSc Kunstmatige Intelligentie", "BSc Informatiekunde", "MSc Information Studies", "MSc Software Engineering", "MSc System and Network Engineering", "MSc Artificial Intelligence", "MSc Logic", "MSc Computational Science", "MSc Computer Science", "MSc Medical Informatics", "MSc Grid Computing", "Other", "Minor programmeren", "Minor Informatica", "Minor Kunstmatige Intelligentie"] db.session.bulk_save_objects(Education(name) for name in education_names) db.session.commit() # Add some default navigation print("* Adding default navigation entries") navigation_entries = [ ('via', 'via', '/via', False, [ ('Nieuws', 'News', '/news/', False, []), ('PimPy', 'PimPy', '/pimpy', False, []), ('Commissies', 'Committees', '/commissie', False, []), ('Admin', 'Admin', '/admin', False, [ ('Navigatie', 'Navigation', '/navigation', False, []), ('Formulieren', 'Forms', '/forms', False, []), ('Redirect', 'Redirect', '/redirect', False, []), ('Users', 'Users', '/users', False, []), ('Groups', 'Groups', '/groups', False, []), ('Files', 'Files', '/files', False, []) ]), ]), ('Activiteiten', 'Activities', '/activities', True, [ ('Activiteiten Archief', 'Activities archive', '/activities/archive', False, []), ('Activiteiten Overzicht', 'Activities overview', '/activities/view', False, []) ]), ('Vacatures', 'Vacancies', '/vacancies/', False, []), ('Tentamenbank', 'Examinations', '/examination', False, []), ('Samenvattingen', 'Summaries', '/summary', False, []) ] _add_navigation(navigation_entries) print("* Adding administrator user") first_name = prompt("\tFirst name") last_name = prompt("\tLast name") email_regex = re.compile("^[^@]+@[^@]+\.[^@]+$") while True: email = prompt("\tEmail") if email_regex.match(email): break print("\tInvalid email address: " + email) while True: passwd_plain = prompt_pass("\tPassword") passwd_plain_rep = prompt_pass("\tRepeat password") if passwd_plain == passwd_plain_rep: break print("\tPasswords do not match") passwd = bcrypt.hashpw(passwd_plain, bcrypt.gensalt()) admin = User( first_name=first_name, last_name=last_name, email=email, password=passwd, education_id=Education.query.first().id) admin.has_paid = True _add_user(admin, True, "A user with email '{}' already exists".format(email)) # Add admin user to administrators group admin_group = Group.query.filter_by(name='administrators').first() admin_group.add_user(admin) db.session.commit() # Grant read/write privilege to administrators group on every module db.session.bulk_save_objects( GroupPermission(module, admin_group.id, 2) for module in app.blueprints.keys()) db.session.commit() print("Done!")
def init_database(connection_url): """ Create and initialize the database engine. This must be done before the session object can be used. This will also attempt to perform any updates to the database schema if the backend support such operations. :param str connection_url: The url for the database connection. :return: The initialized database engine. """ connection_url = normalize_connection_url(connection_url) connection_url = sqlalchemy.engine.url.make_url(connection_url) logger.info("initializing database connection with driver {0}".format( connection_url.drivername)) if connection_url.drivername == 'sqlite': engine = sqlalchemy.create_engine( connection_url, connect_args={'check_same_thread': False}, poolclass=sqlalchemy.pool.StaticPool) sqlalchemy.event.listens_for( engine, 'begin')(lambda conn: conn.execute('BEGIN')) elif connection_url.drivername == 'postgresql': engine = sqlalchemy.create_engine(connection_url) else: raise errors.KingPhisherDatabaseError( 'only sqlite and postgresql database drivers are supported') Session.remove() Session.configure(bind=engine) try: models.Base.metadata.create_all(engine) except sqlalchemy.exc.SQLAlchemyError as error: error_lines = map(lambda line: line.strip(), error.message.split('\n')) raise errors.KingPhisherDatabaseError('SQLAlchemyError: ' + ' '.join(error_lines).strip()) session = Session() set_meta_data('database_driver', connection_url.drivername, session=session) schema_version = (get_meta_data('schema_version', session=session) or models.SCHEMA_VERSION) session.commit() session.close() logger.debug("current database schema version: {0} ({1}current)".format( schema_version, ('' if schema_version == models.SCHEMA_VERSION else 'not '))) if schema_version > models.SCHEMA_VERSION: raise errors.KingPhisherDatabaseError( 'the database schema is for a newer version, automatic downgrades are not supported' ) elif schema_version < models.SCHEMA_VERSION: alembic_config_file = find.find_data_file('alembic.ini') if not alembic_config_file: raise errors.KingPhisherDatabaseError( 'cannot find the alembic.ini configuration file') alembic_directory = find.find_data_directory('alembic') if not alembic_directory: raise errors.KingPhisherDatabaseError( 'cannot find the alembic data directory') config = alembic.config.Config(alembic_config_file) config.config_file_name = alembic_config_file config.set_main_option('script_location', alembic_directory) config.set_main_option('skip_logger_config', 'True') config.set_main_option('sqlalchemy.url', str(connection_url)) logger.warning( "automatically updating the database schema to version {0}".format( models.SCHEMA_VERSION)) try: alembic.command.upgrade(config, 'head') except Exception as error: logger.critical( "database schema upgrade failed with exception: {0}.{1} {2}". format(error.__class__.__module__, error.__class__.__name__, getattr(error, 'message', '')).rstrip()) raise errors.KingPhisherDatabaseError( 'failed to upgrade to the latest database schema') set_meta_data('schema_version', models.SCHEMA_VERSION) logger.debug("connected to {0} database: {1}".format( connection_url.drivername, connection_url.database)) return engine
def init_database(connection_url, extra_init=False): """ Create and initialize the database engine. This must be done before the session object can be used. This will also attempt to perform any updates to the database schema if the backend supports such operations. :param str connection_url: The url for the database connection. :param bool extra_init: Run optional extra dbms-specific initialization logic. :return: The initialized database engine. """ connection_url = normalize_connection_url(connection_url) connection_url = sqlalchemy.engine.url.make_url(connection_url) logger.info("initializing database connection with driver {0}".format( connection_url.drivername)) if connection_url.drivername == 'sqlite': engine = sqlalchemy.create_engine( connection_url, connect_args={'check_same_thread': False}, poolclass=sqlalchemy.pool.StaticPool) sqlalchemy.event.listens_for( engine, 'begin')(lambda conn: conn.execute('BEGIN')) elif connection_url.drivername == 'postgresql': if extra_init: init_database_postgresql(connection_url) engine = sqlalchemy.create_engine( connection_url, connect_args={'client_encoding': 'utf8'}) else: raise errors.KingPhisherDatabaseError( 'only sqlite and postgresql database drivers are supported') Session.remove() Session.configure(bind=engine) inspector = sqlalchemy.inspect(engine) if 'campaigns' not in inspector.get_table_names(): logger.debug('campaigns table not found, creating all new tables') try: models.Base.metadata.create_all(engine) except sqlalchemy.exc.SQLAlchemyError as error: error_lines = (line.strip() for line in error.message.split('\n')) raise errors.KingPhisherDatabaseError( 'SQLAlchemyError: ' + ' '.join(error_lines).strip()) schema_version = get_schema_version(engine) logger.debug("current database schema version: {0} ({1})".format( schema_version, ('latest' if schema_version == models.SCHEMA_VERSION else 'obsolete'))) if 'alembic_version' not in inspector.get_table_names(): logger.debug( 'alembic version table not found, attempting to create and set version' ) init_alembic(engine, schema_version) if schema_version > models.SCHEMA_VERSION: raise errors.KingPhisherDatabaseError( 'the database schema is for a newer version, automatic downgrades are not supported' ) elif schema_version < models.SCHEMA_VERSION: alembic_config_file = find.data_file('alembic.ini') if not alembic_config_file: raise errors.KingPhisherDatabaseError( 'cannot find the alembic.ini configuration file') alembic_directory = find.data_directory('alembic') if not alembic_directory: raise errors.KingPhisherDatabaseError( 'cannot find the alembic data directory') config = alembic.config.Config(alembic_config_file) config.config_file_name = alembic_config_file config.set_main_option('script_location', alembic_directory) config.set_main_option('skip_logger_config', 'True') config.set_main_option('sqlalchemy.url', str(connection_url)) logger.warning( "automatically updating the database schema from version {0} to {1}" .format(schema_version, models.SCHEMA_VERSION)) try: alembic.command.upgrade(config, 'head') except Exception as error: logger.critical( "database schema upgrade failed with exception: {0}.{1} {2}". format(error.__class__.__module__, error.__class__.__name__, getattr(error, 'message', '')).rstrip(), exc_info=True) raise errors.KingPhisherDatabaseError( 'failed to upgrade to the latest database schema') logger.info( "successfully updated the database schema from version {0} to {1}". format(schema_version, models.SCHEMA_VERSION)) # reset it because it may have been altered by alembic Session.remove() Session.configure(bind=engine) set_metadata('database_driver', connection_url.drivername) set_metadata('last_started', datetime.datetime.utcnow()) set_metadata('schema_version', models.SCHEMA_VERSION) logger.debug("connected to {0} database: {1}".format( connection_url.drivername, connection_url.database)) signals.db_initialized.send(connection_url) return engine
def init_database(connection_url, extra_init=False): """ Create and initialize the database engine. This must be done before the session object can be used. This will also attempt to perform any updates to the database schema if the backend supports such operations. :param str connection_url: The url for the database connection. :param bool extra_init: Run optional extra dbms-specific initialization logic. :return: The initialized database engine. """ connection_url = normalize_connection_url(connection_url) connection_url = sqlalchemy.engine.url.make_url(connection_url) logger.info("initializing database connection with driver {0}".format(connection_url.drivername)) if connection_url.drivername == 'sqlite': engine = sqlalchemy.create_engine(connection_url, connect_args={'check_same_thread': False}, poolclass=sqlalchemy.pool.StaticPool) sqlalchemy.event.listens_for(engine, 'begin')(lambda conn: conn.execute('BEGIN')) elif connection_url.drivername == 'postgresql': if extra_init: init_database_postgresql(connection_url) engine = sqlalchemy.create_engine(connection_url) else: raise errors.KingPhisherDatabaseError('only sqlite and postgresql database drivers are supported') Session.remove() Session.configure(bind=engine) inspector = sqlalchemy.inspect(engine) if not 'meta_data' in inspector.get_table_names(): logger.debug('meta_data table not found, creating all new tables') try: models.Base.metadata.create_all(engine) except sqlalchemy.exc.SQLAlchemyError as error: error_lines = (line.strip() for line in error.message.split('\n')) raise errors.KingPhisherDatabaseError('SQLAlchemyError: ' + ' '.join(error_lines).strip()) session = Session() set_meta_data('database_driver', connection_url.drivername, session=session) schema_version = (get_meta_data('schema_version', session=session) or models.SCHEMA_VERSION) session.commit() session.close() logger.debug("current database schema version: {0} ({1})".format(schema_version, ('latest' if schema_version == models.SCHEMA_VERSION else 'obsolete'))) if not 'alembic_version' in inspector.get_table_names(): logger.debug('alembic version table not found, attempting to create and set version') init_alembic(engine, schema_version) if schema_version > models.SCHEMA_VERSION: raise errors.KingPhisherDatabaseError('the database schema is for a newer version, automatic downgrades are not supported') elif schema_version < models.SCHEMA_VERSION: alembic_config_file = find.data_file('alembic.ini') if not alembic_config_file: raise errors.KingPhisherDatabaseError('cannot find the alembic.ini configuration file') alembic_directory = find.data_directory('alembic') if not alembic_directory: raise errors.KingPhisherDatabaseError('cannot find the alembic data directory') config = alembic.config.Config(alembic_config_file) config.config_file_name = alembic_config_file config.set_main_option('script_location', alembic_directory) config.set_main_option('skip_logger_config', 'True') config.set_main_option('sqlalchemy.url', str(connection_url)) logger.warning("automatically updating the database schema from version {0} to {1}".format(schema_version, models.SCHEMA_VERSION)) try: alembic.command.upgrade(config, 'head') except Exception as error: logger.critical("database schema upgrade failed with exception: {0}.{1} {2}".format(error.__class__.__module__, error.__class__.__name__, getattr(error, 'message', '')).rstrip(), exc_info=True) raise errors.KingPhisherDatabaseError('failed to upgrade to the latest database schema') logger.info("successfully updated the database schema from version {0} to {1}".format(schema_version, models.SCHEMA_VERSION)) # reset it because it may have been altered by alembic Session.remove() Session.configure(bind=engine) session = Session() set_meta_data('schema_version', models.SCHEMA_VERSION) logger.debug("connected to {0} database: {1}".format(connection_url.drivername, connection_url.database)) signals.db_initialized.send(connection_url) return engine
def run(self, args) -> int: if (hasattr(args, 'cert') is True and hasattr(args, 'key') is False) or \ (hasattr(args, 'cert') is False and hasattr(args, 'key') is True): self.logger.error("Both the cert and key is required to use SSL") return 1 elif hasattr(args, 'cert') is True and hasattr(args, 'key') is True: cherrypy.server.ssl_certificate = args.cert.name cherrypy.server.ssl_private_key = args.key.name s3_parameters = {} if hasattr(args, 's3_endpoint'): s3_parameters['endpoint_url'] = args.s3_endpoint if hasattr(args, 's3_access_key_id'): s3_parameters['aws_access_key_id'] = args.s3_access_key_id if hasattr(args, 's3_secret_access_key'): s3_parameters['aws_secret_access_key'] = args.s3_secret_access_key self.logger.info("Connecting to S3") s3_client = boto3.resource('s3', **s3_parameters).meta.client try: s3_client.head_bucket(Bucket=args.s3_bucket) except botocore.exceptions.ClientError as e: self.logger.error("Error checking if bucket exists: %s", e) return 1 except botocore.exceptions.NoCredentialsError as e: self.logger.error("Error loading S3 credentials: %s", e) return 1 self.logger.info("Connecting to database") database = Database(db_url=args.db_url, pool_size=args.db_pool_size) database.connect() self.logger.info("Running SQL migrations") config = alembic.config.Config("alembic.ini") config.set_main_option("sqlalchemy.url", database.db_url) config.attributes['connection'] = database.engine.connect() alembic.command.upgrade(config, 'head') http_app = Application(logging_config=None, debug=True) http_app.register_mount(RootMount(http_app, database, args.s3_bucket, s3_client)) http_app.setup() self.logger.info("Running CherryPy Webserver") cherrypy.config.update({ 'global': { 'environment': 'production', 'server.socket_host': str(args.bind_address), 'server.socket_port': args.port, } }) cherrypy.engine.start() cherrypy.engine.block() database.engine.dispose() return 0
def migrate(): config = alembic.config.Config("./alembic.ini") config.set_main_option("sqlalchemy.url", target_db) config.set_main_option("script_location", "./migrate") command.upgrade(config, "head")