def update():
# migrate database to latest revision
db_migrate()
db_upgrade()
# update user roles
model.Role.insert_roles()
model.File.clear_missing_file()
def db_setup():
with app.app_context():
if not os.path.isdir('migrations'):
app.logger.info("The database is being created, to run the server it has to be run again afterwards")
db_init()
db_migrate()
db_upgrade()
return True
return False
def install_packaged_gn_module(module_path, module_code, build):
# install python package and dependencies
subprocess.run(f"pip install -e {module_path}", shell=True, check=True)
# load python package
module_dist = get_dist_from_code(module_code)
if not module_dist:
raise ClickException(f"Unable to load module with code {module_code}")
# add module to database
try:
module_picto = load_entry_point(module_dist, 'gn_module', 'picto')
except ImportError:
module_picto = "fa-puzzle-piece"
module_object = TModules.query.filter_by(module_code=module_code).one()
if not module_object:
module_object = TModules(
module_code=module_code,
module_label=module_code.lower(),
module_path=module_code.lower(),
module_target="_self",
module_picto=module_picto,
active_frontend=True,
active_backend=True,
)
db.session.add(module_object)
else:
module_object.module_picto = module_picto
db.session.merge(module_object)
db.session.commit()
db_upgrade(revision=module_code.lower())
# symlink module in exernal module directory
module_symlink = GN_EXTERNAL_MODULE / module_code.lower()
if os.path.exists(module_symlink):
target = os.readlink(module_symlink)
if os.path.abspath(module_path) != target:
raise ClickException(f"Module symlink has wrong target '{target}'")
else:
os.symlink(os.path.abspath(module_path), module_symlink)
### Frontend
# creation du lien symbolique des assets externes
enable_frontend = create_external_assets_symlink(module_path,
module_code.lower())
install_frontend_dependencies(module_path)
# generation du fichier tsconfig.app.json
tsconfig_app_templating(app=current_app)
# generation du routing du frontend
frontend_routes_templating(app=current_app)
# generation du fichier de configuration du frontend
create_module_config(current_app, module_code, build=False)
if build:
# Rebuild the frontend
build_geonature_front(rebuild_sass=True)
def init():
"Initialize Puffin dependencies"
wait()
db_create()
db_upgrade()
user_create("puffin")
machine_network()
machine_proxy()
machine_mail()
def init():
"Initialize Puffin dependencies"
wait()
db_create()
db_upgrade()
user_create("puffin")
machine_network()
machine_proxy()
machine_mail()
def client():
with tempfile.TemporaryDirectory() as tmpdir:
app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{tmpdir}/db.sqlite"
app.config["FHOST_STORAGE_PATH"] = os.path.join(tmpdir, "up")
app.config["TESTING"] = True
with app.test_client() as client:
with app.app_context():
db_upgrade()
yield client
def bootstrap():
# always run migrations first
db_upgrade()
# always gitbase script to update datasource if it was changed in env var
dbobj = get_or_create_datasource('gitbase',
conf.get('GITBASE_DATABASE_URI'),
allow_run_async=True,
allow_dml=True)
create_datasource_tables(dbobj, conf.get('GITBASE_DB'))
# add metadata data source only in sync mode
if conf.get('SYNC_MODE'):
dbobj = get_or_create_datasource('metadata',
conf.get('METADATA_DATABASE_URI'))
create_datasource_tables(dbobj, conf.get('METADATA_DB'))
# initialize database if empty
users = [u.username for u in security_manager.get_all_users()]
if not conf.get('DEFAULT_USERNAME') in users:
# Create an admin user
role_admin = security_manager.find_role(
security_manager.auth_role_admin)
admin_user = security_manager.add_user(conf.get('DEFAULT_USERNAME'),
os.environ['ADMIN_FIRST_NAME'],
os.environ['ADMIN_LAST_NAME'],
os.environ['ADMIN_EMAIL'],
role_admin,
os.environ['ADMIN_PASSWORD'])
# Create default roles and permissions
utils.get_or_create_main_db()
security_manager.sync_role_definitions()
# set admin user as a current user
g.user = admin_user
# Add dashboards
dashboards_root = '/home/superset/dashboards'
import_dashboard(dashboards_root + '/gitbase/overview.json')
if conf.get('SYNC_MODE'):
import_dashboard(dashboards_root + '/metadata/welcome.json')
import_dashboard(dashboards_root + '/metadata/collaboration.json')
else:
import_dashboard(dashboards_root + '/gitbase/welcome.json')
# set welcome dashboard as a default
set_welcome_dashboard(conf.get('DEFAULT_DASHBOARD_ID'), admin_user)
def init_db():
"""
Create the database and populate the required data.
"""
db_url = current_app.config["SQLALCHEMY_DATABASE_URI"]
try:
if database_exists(db_url):
if "user" in db.metadata.tables.keys():
print("Database already exists, check your installation.")
sys.exit(1)
else:
create_database(db_url)
except exc.OperationalError:
create_database(db_url)
db_upgrade()
# User roles
# TODO: This list should probably be in the models file
user_datastore.create_role(name="Admin", description="Site Administrator")
user_datastore.create_role(name="Member", description="Member")
db.session.commit()
def session(app, db, fresh_db, monkeypatch):
"""Creates a new database session for a test."""
connection = db.engine.connect()
if not fresh_db:
transaction = connection.begin()
options = dict(bind=connection, binds={}, autoflush=False)
session = db.create_scoped_session(options=options)
old_uri = app.config['SQLALCHEMY_DATABASE_URI']
if not fresh_db:
session.begin_nested()
# then each time that SAVEPOINT ends, reopen it
@sqlalchemy.event.listens_for(session, 'after_transaction_end')
def restart_savepoint(session, transaction):
if transaction.nested and not transaction._parent.nested:
# ensure that state is expired the way
# session.commit() at the top level normally does
# (optional step)
session.expire_all()
session.begin_nested()
if fresh_db:
with app.app_context():
from flask_migrate import Migrate
from flask_migrate import upgrade as db_upgrade
logging.disable(logging.ERROR)
Migrate(app, db)
db_upgrade()
logging.disable(logging.NOTSET)
manage.seed_force(psef.models.db)
manage.test_data(psef.models.db)
try:
psef.models.validator._update_session(session)
with monkeypatch.context() as context:
context.setattr(psef.models.db, 'session', session)
if not fresh_db:
context.setattr(session, 'remove', lambda: None)
yield session
finally:
if not fresh_db:
sqlalchemy.event.remove(
session, 'after_transaction_end', restart_savepoint
)
transaction.rollback()
try:
session.remove()
connection.close()
except:
import traceback
traceback.print_exc(file=sys.stderr)
raise
if fresh_db:
db.engine.dispose()
db.drop_all()
drop_database(old_uri)
account_type=0,
password_hash=
'pbkdf2:sha256:150000$Sn5LeTtv$b9bfc8a77bc8e232c90f494dc09c64c2b9604901b3b34a1ea6d03ebea3083cdf'
)
db.session.add(user)
db.session.commit()
except:
app.logger.warn('Error, no user table')
with app.app_context():
from flask_migrate import init as db_init, migrate as db_migrate, upgrade as db_upgrade
app.logger.info('Initializing database...')
db_init()
db_migrate(message='Initializing database')
db_upgrade()
app.logger.info('Database initialized')
app.logger.info('Creating root user')
user = User(
username='******',
account_type=0,
password_hash=
'pbkdf2:sha256:150000$Sn5LeTtv$b9bfc8a77bc8e232c90f494dc09c64c2b9604901b3b34a1ea6d03ebea3083cdf'
)
db.session.add(user)
db.session.commit()
from app import routes, models, errors, permissions
# Connection handshake with Vision System
def install_packaged_gn_module(module_path, module_code, build):
# install python package and dependencies
subprocess.run(f"pip install -e '{module_path}'", shell=True, check=True)
# refresh list of entry points
importlib.reload(site)
for entry in sys.path:
pkg_resources.working_set.add_entry(entry)
# load python package
module_dist = get_dist_from_code(module_code)
if not module_dist:
raise ClickException(f"Unable to load module with code {module_code}")
# add module to database
try:
module_picto = load_entry_point(module_dist, 'gn_module', 'picto')
except ImportError:
module_picto = "fa-puzzle-piece"
try:
module_object = TModules.query.filter_by(module_code=module_code).one()
module_object.module_picto = module_picto
db.session.merge(module_object)
except sa_exc.NoResultFound:
module_object = TModules(
module_code=module_code,
module_label=module_code.lower(),
module_path=module_code.lower(),
module_target="_self",
module_picto=module_picto,
active_frontend=True,
active_backend=True,
)
db.session.add(module_object)
db.session.commit()
info = get_entry_info(module_dist, 'gn_module', 'migrations')
if info is not None:
try:
alembic_branch = load_entry_point(module_dist, 'gn_module',
'alembic_branch')
except ImportError:
alembic_branch = module_code.lower()
db_upgrade(revision=alembic_branch + '@head')
else:
log.info(
"Module do not provide any migration files, skipping database upgrade."
)
# symlink module in exernal module directory
module_symlink = GN_EXTERNAL_MODULE / module_code.lower()
if os.path.exists(module_symlink):
target = os.readlink(module_symlink)
if os.path.realpath(module_path) != os.path.realpath(target):
raise ClickException(
f"Module symlink has wrong target: '{target}'")
else:
os.symlink(os.path.abspath(module_path), module_symlink)
# creation du fichier conf_gn_module.toml
gn_module_register_config(module_code)
### Frontend
# creation du lien symbolique des assets externes
enable_frontend = create_external_assets_symlink(module_path,
module_code.lower())
install_frontend_dependencies(module_path)
# generation du fichier tsconfig.app.json
tsconfig_app_templating(app=current_app)
# generation du routing du frontend
frontend_routes_templating(app=current_app)
# generation du fichier de configuration du frontend
create_module_config(current_app, module_code, build=False)
if build:
# Rebuild the frontend
build_geonature_front(rebuild_sass=True)
def _upgrade_db(program_dir):
migrations_path = os.path.join(program_dir, "migrations")
db_upgrade(directory=migrations_path)
def run(**kwargs):
""" Run the Gunicorn worker """
kwargs['reload'] = kwargs['debug']
kwargs['preload'] = not kwargs['debug']
APP.debug = kwargs['debug']
if kwargs['collect_static']:
subprocess.check_call('./_deps_collectstatic.sh',
cwd=project_root().strpath)
if kwargs['timeout'] != 0:
start_time = time.time()
db_engine = create_engine(
get_db_uri(),
connect_args=dict(connect_timeout=2),
)
db_conn = None
mq_conn = None
while time.time() - start_time < kwargs['timeout']:
try:
if db_conn is None or db_conn.closed:
db_conn = db_engine.connect()
except OperationalError:
time.sleep(2)
continue
try:
if mq_conn is None:
mq_conn = get_pika_conn()
except AMQPError:
time.sleep(2)
continue
break
if db_conn is None or db_conn.closed:
stderr.write("Timed out waiting for the database to be ready\n")
return 1
if mq_conn is None:
stderr.write("Timed out waiting for RabbitMQ to be ready\n")
return 1
# Setup the exchange
channel = mq_conn.channel()
channel.exchange_declare(exchange='dockci.job', type='topic')
channel.exchange_declare(exchange='dockci.queue', type='topic')
channel.queue_declare(queue='dockci.agent')
channel.queue_bind(exchange='dockci.queue',
queue='dockci.agent',
routing_key='*')
mq_conn.close()
if kwargs['db_migrate']:
db_upgrade( # doesn't return anything
local(__file__).dirpath().join('../../alembic').strpath
)
else:
# Migrate will init the app for us
app_init()
GunicornWrapper(kwargs).run()
