def db():
"""Get a db session"""
global _db
if _db is None:
_db = get_session('sqlite:///:memory:')
return _db
def reindex_packages_from_store(
config: Config,
channel_name: str,
user_id: bytes,
):
"""Reindex packages from files in the package store"""
db = get_session(config.sqlalchemy_database_url)
dao = Dao(db)
pkgstore = config.get_package_store()
all_files = pkgstore.list_files(channel_name)
pkg_files = [f for f in all_files if f.endswith(".tar.bz2")]
channel = dao.get_channel(channel_name)
if channel:
for package in channel.packages:
db.delete(package)
db.commit()
else:
data = rest_models.Channel(
name=channel_name, description="re-indexed from files", private=True
)
channel = dao.create_channel(data, user_id, authorization.OWNER)
for fname in pkg_files:
fid = pkgstore.serve_path(channel_name, fname)
handle_file(channel_name, fname, fid, dao, user_id)
update_indexes(dao, pkgstore, channel_name)
def get_db_manager():
db = get_session(config.sqlalchemy_database_url)
try:
yield db
finally:
db.close()
def db_extra(database_url):
"""a separate session for db connection
Use only for tests that require two sessions concurrently.
For most cases you will want to use the db fixture (from quetz.testing.fixtures)"""
session = get_session(database_url)
yield session
session.close()
def db_cleanup(config):
# we can't use the db fixture for cleaning up because
# it automatically rollsback all operations
yield
from quetz.database import get_session
db = get_session(config.sqlalchemy_database_url)
user = db.query(User).one_or_none()
if user:
db.delete(user)
db.commit()
def init_db(
path: str = typer.Argument(None, help="The path of the deployment"),
):
"""init database and fill users from config file [users] sections"""
logger.info("Initializing database")
config_file = _get_config(path)
config = Config(config_file)
os.chdir(path)
db = get_session(config.sqlalchemy_database_url)
_init_db(db, config)
def start_supervisor_daemon(path, num_procs=None):
from quetz.jobs.runner import Supervisor
from quetz.tasks.workers import get_worker
configure_logger(loggers=("quetz", ))
config = _get_config(path)
manager = get_worker(config, num_procs=num_procs)
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
supervisor = Supervisor(db, manager)
try:
supervisor.run()
except KeyboardInterrupt:
logger.info("stopping supervisor")
finally:
db.close()
def add_user_roles(
path: str = typer.Argument(None, help="The path of the deployment"), ):
"""Set user roles according to the values from config file [users] sections.
This command will assign roles only if they were not set before
(for example using API or an earlier setting in config). The only exception
is that users who already have a role defined as default_role will have a
new role set from the config.
This command will NOT remove roles of existing roles, even if they were
removed from the config file.
"""
logger.info("setting up user roles")
config = _get_config(path)
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
_set_user_roles(db, config)
def job_wrapper(func, api_key, browser_session, config, **kwargs):
# database connections etc. are not serializable
# so we need to recreate them in the process.
# This allows us to manage database connectivity prior
# to running a job.
import logging
import os
from quetz.authorization import Rules
from quetz.config import configure_logger
from quetz.dao import Dao
from quetz.database import get_session
from quetz.deps import get_remote_session
pkgstore = config.get_package_store()
db = get_session(config.sqlalchemy_database_url)
dao = Dao(db)
auth = Rules(api_key, browser_session, db)
session = get_remote_session()
configure_logger(config)
logger = logging.getLogger("quetz")
logger.debug(
f"evaluating function {func} in a subprocess task with pid {os.getpid()}"
)
extra_kwargs = prepare_arguments(
func,
dao=dao,
auth=auth,
session=session,
config=config,
pkgstore=pkgstore,
)
kwargs.update(extra_kwargs)
func(**kwargs)
def get_test_config():
c = {
'github': {
'client_id': '',
'client_secret': ''
},
'sqlalchemy': {
'database_url': ''
},
'session': {
'secret': 'abcdefg',
'https_only': False
}
}
global _db
if _db is None:
_db = get_session('sqlite:///:memory:')
conf = Config(config=c, db_session=_db)
return conf
def job_wrapper(
func: Union[Callable, bytes], api_key, browser_session, config, **kwargs
):
# database connections etc. are not serializable
# so we need to recreate them in the process.
# This allows us to manage database connectivity prior
# to running a job.
import pickle
from quetz.authorization import Rules
from quetz.dao import Dao
from quetz.database import get_session
from quetz.deps import get_remote_session
pkgstore = config.get_package_store()
db = get_session(config.sqlalchemy_database_url)
dao = Dao(db)
auth = Rules(api_key, browser_session, db)
session = get_remote_session()
callable_f: Callable = pickle.loads(func) if isinstance(func, bytes) else func
extra_kwargs = prepare_arguments(
callable_f,
dao=dao,
auth=auth,
session=session,
config=config,
pkgstore=pkgstore,
)
kwargs.update(extra_kwargs)
try:
callable_f(**kwargs)
finally:
db.close()
def watch_job_queue(
path: str = typer.Argument(None, help="Path to the plugin folder"),
num_procs: Optional[int] = typer.Option(
None, help="Number of processes to use. Default: number of CPU cores"),
) -> None:
import time
configure_logger(loggers=("quetz", ))
from quetz.jobs.runner import check_status, run_jobs, run_tasks
from quetz.tasks.workers import SubprocessWorker
config = _get_config(path)
manager = SubprocessWorker("", {}, config, {'max_workers': num_procs})
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
try:
while True:
run_jobs(db)
run_tasks(db, manager)
check_status(db)
time.sleep(5)
except KeyboardInterrupt:
db.close()
def create(
path: str = typer.Argument(
None,
help=("The directory in which the deployment will be created "
"(will be created if does not exist)"),
),
copy_conf: str = typer.Option(
None, help="The configuration to copy from (e.g. dev_config.toml)"),
create_conf: bool = typer.Option(
False,
help="Enable/disable creation of a default configuration file",
),
delete: bool = typer.Option(
False,
help="Delete the the deployment if it exists. "
"Must be specified with --copy-conf or --create-conf",
),
exists_ok: bool = typer.Option(
False, help="Skip the creation if deployment already exists."),
dev: bool = typer.Option(
False,
help=("Enable/disable dev mode "
"(fills the database with test data and allows http access)"),
),
):
"""Create a new Quetz deployment."""
logger.info(f"creating new deployment in path {path}")
deployment_folder = Path(path).resolve()
config_file = deployment_folder / "config.toml"
if _is_deployment(deployment_folder):
if exists_ok:
logger.info(
f'Quetz deployment already exists at {deployment_folder}.\n'
f'Skipping creation.')
return
if delete and (copy_conf or create_conf):
shutil.rmtree(deployment_folder)
else:
typer.echo(
'Use the start command to start a deployment '
'or specify --delete with --copy-conf or --create-conf.',
err=True,
)
raise typer.Abort()
deployment_folder.mkdir(parents=True, exist_ok=True)
# only authorize path with a config file to avoid deletion of unexpected files
# when deleting Quetz instance
if any(f != config_file for f in deployment_folder.iterdir()):
typer.echo(
f'Quetz deployment not allowed at {path}.\n'
'The path should not contain more than the configuration file.',
err=True,
)
raise typer.Abort()
if not config_file.exists() and not create_conf and not copy_conf:
typer.echo(
'No configuration file provided.\n'
'Use --create-conf or --copy-conf to produce a config file.',
err=True,
)
raise typer.Abort()
if copy_conf:
if not os.path.exists(copy_conf):
typer.echo(f'Config file to copy does not exist {copy_conf}.',
err=True)
raise typer.Abort()
typer.echo(f"Copying config file from {copy_conf} to {config_file}")
shutil.copyfile(copy_conf, config_file)
if not config_file.exists() and create_conf:
https = 'false' if dev else 'true'
conf = create_config(https=https)
with open(config_file, 'w') as f:
f.write(conf)
os.environ[_env_prefix + _env_config_file] = str(config_file.resolve())
config = Config(str(config_file))
deployment_folder.joinpath('channels').mkdir(exist_ok=True)
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
_run_migrations(config.sqlalchemy_database_url)
if dev:
_fill_test_database(db)
_set_user_roles(db, config)
def _fill_test_database(database_url: str) -> NoReturn:
""" Create dummy users and channels to allow further testing in dev mode."""
db = get_session(database_url)
testUsers = []
try:
for index, username in enumerate(['alice', 'bob', 'carol', 'dave']):
user = User(id=uuid.uuid4().bytes, username=username)
identity = Identity(
provider='dummy',
identity_id=str(index),
username=username,
)
profile = Profile(name=username.capitalize(),
avatar_url='/avatar.jpg')
user.identities.append(identity)
user.profile = profile
db.add(user)
testUsers.append(user)
for channel_index in range(3):
channel = Channel(
name=f'channel{channel_index}',
description=f'Description of channel{channel_index}',
private=False)
for package_index in range(random.randint(5, 10)):
package = Package(
name=f'package{package_index}',
description=f'Description of package{package_index}')
channel.packages.append(package)
test_user = testUsers[random.randint(0, len(testUsers) - 1)]
package_member = PackageMember(package=package,
channel=channel,
user=test_user,
role='owner')
db.add(package_member)
if channel_index == 0:
package = Package(name='xtensor',
description='Description of xtensor')
channel.packages.append(package)
test_user = testUsers[random.randint(0, len(testUsers) - 1)]
package_member = PackageMember(package=package,
channel=channel,
user=test_user,
role='owner')
db.add(package_member)
# create API key
key = uuid.uuid4().hex
key_user = User(id=uuid.uuid4().bytes)
api_key = ApiKey(key=key,
description='test API key',
user=test_user,
owner=test_user)
db.add(api_key)
print(
f'Test API key created for user "{test_user.username}": {key}'
)
key_package_member = PackageMember(user=key_user,
channel_name=channel.name,
package_name=package.name,
role='maintainer')
db.add(key_package_member)
db.add(channel)
channel_member = ChannelMember(channel=channel,
user=testUsers[random.randint(
0,
len(testUsers) - 1)],
role='owner')
db.add(channel_member)
db.commit()
finally:
db.close()
def create(
path: str = typer.Argument(
None,
help=(
"The directory in which the deployment will be created "
"(will be created if does not exist)"
),
),
config_file_name: str = typer.Option(
"config.toml", help="The configuration file name expected in the provided path"
),
copy_conf: str = typer.Option(
None, help="The configuration to copy from (e.g. dev_config.toml)"
),
create_conf: bool = typer.Option(
False,
help="Enable/disable creation of a default configuration file",
),
dev: bool = typer.Option(
False,
help=(
"Enable/disable dev mode "
"(fills the database with test data and allows http access)"
),
),
):
"""Create a new Quetz deployment."""
logger.info(f"creating new deployment in path {path}")
abs_path = os.path.abspath(path)
config_file = os.path.join(path, config_file_name)
deployments = _get_deployments()
if os.path.exists(path) and abs_path in deployments:
delete_ = typer.confirm(f'Quetz deployment exists at {path}.\nOverwrite it?')
if delete_:
delete(path, force=True)
del deployments[abs_path]
else:
typer.echo('Use the start command to start a deployment.', err=True)
raise typer.Abort()
Path(path).mkdir(parents=True)
# only authorize path with a config file to avoid deletion of unexpected files
# when deleting Quetz instance
if not all(f == config_file_name for f in os.listdir(path)):
typer.echo(
f'Quetz deployment not allowed at {path}.\n'
'The path should not contain more than the configuration file.',
err=True,
)
raise typer.Abort()
if not os.path.exists(config_file) and not (create_conf or copy_conf):
typer.echo(
'No configuration file provided.\n'
'Use --create-conf or --copy-conf to produce a config file.',
err=True,
)
raise typer.Abort()
if copy_conf:
if not os.path.exists(copy_conf):
typer.echo(f'Config file to copy does not exist {copy_conf}.', err=True)
raise typer.Abort()
typer.echo(f"Copying config file from {copy_conf} to {config_file}")
shutil.copyfile(copy_conf, config_file)
if not os.path.exists(config_file) and create_conf:
if dev:
https = 'false'
else:
https = 'true'
conf = create_config(https=https)
with open(config_file, 'w') as f:
f.write(conf)
os.environ[_env_prefix + _env_config_file] = config_file
config = Config(config_file)
os.chdir(path)
Path('channels').mkdir()
db = get_session(config.sqlalchemy_database_url)
_init_db(db, config)
if dev:
_fill_test_database(db)
_store_deployment(abs_path, config_file_name)
def init_test_db():
config = Config()
init_db(config.sqlalchemy_database_url)
db = get_session(config.sqlalchemy_database_url)
testUsers = []
try:
for index, username in enumerate(['alice', 'bob', 'carol', 'dave']):
user = User(id=uuid.uuid4().bytes, username=username)
identity = Identity(
provider='dummy',
identity_id=str(index),
username=username,
)
profile = Profile(name=username.capitalize(),
avatar_url='/avatar.jpg')
user.identities.append(identity)
user.profile = profile
db.add(user)
testUsers.append(user)
for channel_index in range(30):
channel = Channel(
name=f'channel{channel_index}',
description=f'Description of channel{channel_index}',
private=False,
)
for package_index in range(random.randint(5, 100)):
package = Package(
name=f'package{package_index}',
description=f'Description of package{package_index}',
)
channel.packages.append(package)
test_user = testUsers[random.randint(0, len(testUsers) - 1)]
package_member = PackageMember(package=package,
channel=channel,
user=test_user,
role='owner')
db.add(package_member)
if channel_index == 0:
package = Package(name='xtensor',
description='Description of xtensor')
channel.packages.append(package)
test_user = testUsers[random.randint(0, len(testUsers) - 1)]
package_member = PackageMember(package=package,
channel=channel,
user=test_user,
role='owner')
db.add(package_member)
# create API key
key = 'E_KaBFstCKI9hTdPM7DQq56GglRHf2HW7tQtq6si370'
key_user = User(id=uuid.uuid4().bytes)
api_key = ApiKey(key=key,
description='test API key',
user=key_user,
owner=test_user)
db.add(api_key)
key_package_member = PackageMember(
user=key_user,
channel_name=channel.name,
package_name=package.name,
role='maintainer',
)
db.add(key_package_member)
db.add(channel)
channel_member = ChannelMember(
channel=channel,
user=testUsers[random.randint(0,
len(testUsers) - 1)],
role='owner',
)
db.add(channel_member)
db.commit()
finally:
db.close()
def job_wrapper(
func: Union[Callable, bytes],
config,
task_id=None,
exc_passthrou=False,
**kwargs,
):
# database connections etc. are not serializable
# so we need to recreate them in the process.
# This allows us to manage database connectivity prior
# to running a job.
import logging
import pickle
from quetz.authorization import Rules
from quetz.config import configure_logger
from quetz.dao import Dao
from quetz.database import get_session
from quetz.deps import get_remote_session
configure_logger(config)
logger = logging.getLogger("quetz.worker")
pkgstore = kwargs.pop("pkgstore", None)
db = kwargs.pop("db", None)
dao = kwargs.pop("dao", None)
auth = kwargs.pop("auth", None)
session = kwargs.pop("session", None)
if db:
close_session = False
elif dao:
db = dao.db
close_session = False
else:
db = get_session(config.sqlalchemy_database_url)
close_session = True
user_id: Optional[str]
if task_id:
task = db.query(Task).filter(Task.id == task_id).one_or_none()
# take extra arguments from job definition
if task.job.extra_args:
job_extra_args = json.loads(task.job.extra_args)
kwargs.update(job_extra_args)
if task.job.owner_id:
user_id = str(uuid.UUID(bytes=task.job.owner_id))
else:
user_id = None
else:
task = None
user_id = None
if not pkgstore:
pkgstore = config.get_package_store()
dao = Dao(db)
if not auth:
browser_session: Dict[str, str] = {}
api_key = None
if user_id:
browser_session['user_id'] = user_id
auth = Rules(api_key, browser_session, db)
if not session:
session = get_remote_session()
if task:
task.status = TaskStatus.running
task.job.status = JobStatus.running
db.commit()
callable_f: Callable = pickle.loads(func) if isinstance(func,
bytes) else func
extra_kwargs = prepare_arguments(
callable_f,
dao=dao,
auth=auth,
session=session,
config=config,
pkgstore=pkgstore,
user_id=user_id,
)
kwargs.update(extra_kwargs)
try:
callable_f(**kwargs)
except Exception as exc:
if task:
task.status = TaskStatus.failed
logger.error(
f"exception occurred when evaluating function {callable_f.__name__}:{exc}"
)
if exc_passthrou:
raise exc
else:
if task:
task.status = TaskStatus.success
finally:
db.commit()
if close_session:
db.close()
