def update(): # migrate database to latest revision db_migrate() db_upgrade() # update user roles model.Role.insert_roles() model.File.clear_missing_file()
def db_setup(): with app.app_context(): if not os.path.isdir('migrations'): app.logger.info("The database is being created, to run the server it has to be run again afterwards") db_init() db_migrate() db_upgrade() return True return False
def install_packaged_gn_module(module_path, module_code, build): # install python package and dependencies subprocess.run(f"pip install -e {module_path}", shell=True, check=True) # load python package module_dist = get_dist_from_code(module_code) if not module_dist: raise ClickException(f"Unable to load module with code {module_code}") # add module to database try: module_picto = load_entry_point(module_dist, 'gn_module', 'picto') except ImportError: module_picto = "fa-puzzle-piece" module_object = TModules.query.filter_by(module_code=module_code).one() if not module_object: module_object = TModules( module_code=module_code, module_label=module_code.lower(), module_path=module_code.lower(), module_target="_self", module_picto=module_picto, active_frontend=True, active_backend=True, ) db.session.add(module_object) else: module_object.module_picto = module_picto db.session.merge(module_object) db.session.commit() db_upgrade(revision=module_code.lower()) # symlink module in exernal module directory module_symlink = GN_EXTERNAL_MODULE / module_code.lower() if os.path.exists(module_symlink): target = os.readlink(module_symlink) if os.path.abspath(module_path) != target: raise ClickException(f"Module symlink has wrong target '{target}'") else: os.symlink(os.path.abspath(module_path), module_symlink) ### Frontend # creation du lien symbolique des assets externes enable_frontend = create_external_assets_symlink(module_path, module_code.lower()) install_frontend_dependencies(module_path) # generation du fichier tsconfig.app.json tsconfig_app_templating(app=current_app) # generation du routing du frontend frontend_routes_templating(app=current_app) # generation du fichier de configuration du frontend create_module_config(current_app, module_code, build=False) if build: # Rebuild the frontend build_geonature_front(rebuild_sass=True)
def init(): "Initialize Puffin dependencies" wait() db_create() db_upgrade() user_create("puffin") machine_network() machine_proxy() machine_mail()
def init(): "Initialize Puffin dependencies" wait() db_create() db_upgrade() user_create("puffin") machine_network() machine_proxy() machine_mail()
def client(): with tempfile.TemporaryDirectory() as tmpdir: app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{tmpdir}/db.sqlite" app.config["FHOST_STORAGE_PATH"] = os.path.join(tmpdir, "up") app.config["TESTING"] = True with app.test_client() as client: with app.app_context(): db_upgrade() yield client
def bootstrap(): # always run migrations first db_upgrade() # always gitbase script to update datasource if it was changed in env var dbobj = get_or_create_datasource('gitbase', conf.get('GITBASE_DATABASE_URI'), allow_run_async=True, allow_dml=True) create_datasource_tables(dbobj, conf.get('GITBASE_DB')) # add metadata data source only in sync mode if conf.get('SYNC_MODE'): dbobj = get_or_create_datasource('metadata', conf.get('METADATA_DATABASE_URI')) create_datasource_tables(dbobj, conf.get('METADATA_DB')) # initialize database if empty users = [u.username for u in security_manager.get_all_users()] if not conf.get('DEFAULT_USERNAME') in users: # Create an admin user role_admin = security_manager.find_role( security_manager.auth_role_admin) admin_user = security_manager.add_user(conf.get('DEFAULT_USERNAME'), os.environ['ADMIN_FIRST_NAME'], os.environ['ADMIN_LAST_NAME'], os.environ['ADMIN_EMAIL'], role_admin, os.environ['ADMIN_PASSWORD']) # Create default roles and permissions utils.get_or_create_main_db() security_manager.sync_role_definitions() # set admin user as a current user g.user = admin_user # Add dashboards dashboards_root = '/home/superset/dashboards' import_dashboard(dashboards_root + '/gitbase/overview.json') if conf.get('SYNC_MODE'): import_dashboard(dashboards_root + '/metadata/welcome.json') import_dashboard(dashboards_root + '/metadata/collaboration.json') else: import_dashboard(dashboards_root + '/gitbase/welcome.json') # set welcome dashboard as a default set_welcome_dashboard(conf.get('DEFAULT_DASHBOARD_ID'), admin_user)
def init_db(): """ Create the database and populate the required data. """ db_url = current_app.config["SQLALCHEMY_DATABASE_URI"] try: if database_exists(db_url): if "user" in db.metadata.tables.keys(): print("Database already exists, check your installation.") sys.exit(1) else: create_database(db_url) except exc.OperationalError: create_database(db_url) db_upgrade() # User roles # TODO: This list should probably be in the models file user_datastore.create_role(name="Admin", description="Site Administrator") user_datastore.create_role(name="Member", description="Member") db.session.commit()
def session(app, db, fresh_db, monkeypatch): """Creates a new database session for a test.""" connection = db.engine.connect() if not fresh_db: transaction = connection.begin() options = dict(bind=connection, binds={}, autoflush=False) session = db.create_scoped_session(options=options) old_uri = app.config['SQLALCHEMY_DATABASE_URI'] if not fresh_db: session.begin_nested() # then each time that SAVEPOINT ends, reopen it @sqlalchemy.event.listens_for(session, 'after_transaction_end') def restart_savepoint(session, transaction): if transaction.nested and not transaction._parent.nested: # ensure that state is expired the way # session.commit() at the top level normally does # (optional step) session.expire_all() session.begin_nested() if fresh_db: with app.app_context(): from flask_migrate import Migrate from flask_migrate import upgrade as db_upgrade logging.disable(logging.ERROR) Migrate(app, db) db_upgrade() logging.disable(logging.NOTSET) manage.seed_force(psef.models.db) manage.test_data(psef.models.db) try: psef.models.validator._update_session(session) with monkeypatch.context() as context: context.setattr(psef.models.db, 'session', session) if not fresh_db: context.setattr(session, 'remove', lambda: None) yield session finally: if not fresh_db: sqlalchemy.event.remove( session, 'after_transaction_end', restart_savepoint ) transaction.rollback() try: session.remove() connection.close() except: import traceback traceback.print_exc(file=sys.stderr) raise if fresh_db: db.engine.dispose() db.drop_all() drop_database(old_uri)
account_type=0, password_hash= 'pbkdf2:sha256:150000$Sn5LeTtv$b9bfc8a77bc8e232c90f494dc09c64c2b9604901b3b34a1ea6d03ebea3083cdf' ) db.session.add(user) db.session.commit() except: app.logger.warn('Error, no user table') with app.app_context(): from flask_migrate import init as db_init, migrate as db_migrate, upgrade as db_upgrade app.logger.info('Initializing database...') db_init() db_migrate(message='Initializing database') db_upgrade() app.logger.info('Database initialized') app.logger.info('Creating root user') user = User( username='******', account_type=0, password_hash= 'pbkdf2:sha256:150000$Sn5LeTtv$b9bfc8a77bc8e232c90f494dc09c64c2b9604901b3b34a1ea6d03ebea3083cdf' ) db.session.add(user) db.session.commit() from app import routes, models, errors, permissions # Connection handshake with Vision System
def install_packaged_gn_module(module_path, module_code, build): # install python package and dependencies subprocess.run(f"pip install -e '{module_path}'", shell=True, check=True) # refresh list of entry points importlib.reload(site) for entry in sys.path: pkg_resources.working_set.add_entry(entry) # load python package module_dist = get_dist_from_code(module_code) if not module_dist: raise ClickException(f"Unable to load module with code {module_code}") # add module to database try: module_picto = load_entry_point(module_dist, 'gn_module', 'picto') except ImportError: module_picto = "fa-puzzle-piece" try: module_object = TModules.query.filter_by(module_code=module_code).one() module_object.module_picto = module_picto db.session.merge(module_object) except sa_exc.NoResultFound: module_object = TModules( module_code=module_code, module_label=module_code.lower(), module_path=module_code.lower(), module_target="_self", module_picto=module_picto, active_frontend=True, active_backend=True, ) db.session.add(module_object) db.session.commit() info = get_entry_info(module_dist, 'gn_module', 'migrations') if info is not None: try: alembic_branch = load_entry_point(module_dist, 'gn_module', 'alembic_branch') except ImportError: alembic_branch = module_code.lower() db_upgrade(revision=alembic_branch + '@head') else: log.info( "Module do not provide any migration files, skipping database upgrade." ) # symlink module in exernal module directory module_symlink = GN_EXTERNAL_MODULE / module_code.lower() if os.path.exists(module_symlink): target = os.readlink(module_symlink) if os.path.realpath(module_path) != os.path.realpath(target): raise ClickException( f"Module symlink has wrong target: '{target}'") else: os.symlink(os.path.abspath(module_path), module_symlink) # creation du fichier conf_gn_module.toml gn_module_register_config(module_code) ### Frontend # creation du lien symbolique des assets externes enable_frontend = create_external_assets_symlink(module_path, module_code.lower()) install_frontend_dependencies(module_path) # generation du fichier tsconfig.app.json tsconfig_app_templating(app=current_app) # generation du routing du frontend frontend_routes_templating(app=current_app) # generation du fichier de configuration du frontend create_module_config(current_app, module_code, build=False) if build: # Rebuild the frontend build_geonature_front(rebuild_sass=True)
def _upgrade_db(program_dir): migrations_path = os.path.join(program_dir, "migrations") db_upgrade(directory=migrations_path)
def run(**kwargs): """ Run the Gunicorn worker """ kwargs['reload'] = kwargs['debug'] kwargs['preload'] = not kwargs['debug'] APP.debug = kwargs['debug'] if kwargs['collect_static']: subprocess.check_call('./_deps_collectstatic.sh', cwd=project_root().strpath) if kwargs['timeout'] != 0: start_time = time.time() db_engine = create_engine( get_db_uri(), connect_args=dict(connect_timeout=2), ) db_conn = None mq_conn = None while time.time() - start_time < kwargs['timeout']: try: if db_conn is None or db_conn.closed: db_conn = db_engine.connect() except OperationalError: time.sleep(2) continue try: if mq_conn is None: mq_conn = get_pika_conn() except AMQPError: time.sleep(2) continue break if db_conn is None or db_conn.closed: stderr.write("Timed out waiting for the database to be ready\n") return 1 if mq_conn is None: stderr.write("Timed out waiting for RabbitMQ to be ready\n") return 1 # Setup the exchange channel = mq_conn.channel() channel.exchange_declare(exchange='dockci.job', type='topic') channel.exchange_declare(exchange='dockci.queue', type='topic') channel.queue_declare(queue='dockci.agent') channel.queue_bind(exchange='dockci.queue', queue='dockci.agent', routing_key='*') mq_conn.close() if kwargs['db_migrate']: db_upgrade( # doesn't return anything local(__file__).dirpath().join('../../alembic').strpath ) else: # Migrate will init the app for us app_init() GunicornWrapper(kwargs).run()