def quicksetup(
ctx, non_interactive, profile, email, first_name, last_name, institution, db_engine, db_backend, db_host, db_port,
db_name, db_username, db_password, su_db_name, su_db_username, su_db_password, broker_protocol, broker_username,
broker_password, broker_host, broker_port, broker_virtual_host, repository
):
"""Setup a new profile in a fully automated fashion."""
# pylint: disable=too-many-arguments,too-many-locals
from aiida.manage.external.postgres import Postgres, manual_setup_instructions
dbinfo_su = {
'host': db_host,
'port': db_port,
'user': su_db_username,
'password': su_db_password,
}
postgres = Postgres(interactive=not non_interactive, quiet=False, dbinfo=dbinfo_su)
if not postgres.is_connected:
echo.echo_critical('failed to determine the PostgreSQL setup')
try:
db_username, db_name = postgres.create_dbuser_db_safe(dbname=db_name, dbuser=db_username, dbpass=db_password)
except Exception as exception:
echo.echo_error(
'\n'.join([
'Oops! quicksetup was unable to create the AiiDA database for you.',
'See `verdi quicksetup -h` for how to specify non-standard parameters for the postgresql connection.\n'
'Alternatively, create the AiiDA database yourself: ',
manual_setup_instructions(dbuser=su_db_username,
dbname=su_db_name), '', 'and then use `verdi setup` instead', ''
])
)
raise exception
# The contextual defaults or `verdi setup` are not being called when `invoking`, so we have to explicitly define
# them here, even though the `verdi setup` command would populate those when called from the command line.
setup_parameters = {
'non_interactive': non_interactive,
'profile': profile,
'email': email,
'first_name': first_name,
'last_name': last_name,
'institution': institution,
'db_engine': db_engine,
'db_backend': db_backend,
'db_name': db_name,
# from now on we connect as the AiiDA DB user, which may be forbidden when going via sockets
'db_host': postgres.host_for_psycopg2,
'db_port': postgres.port_for_psycopg2,
'db_username': db_username,
'db_password': db_password,
'broker_protocol': broker_protocol,
'broker_username': broker_username,
'broker_password': broker_password,
'broker_host': broker_host,
'broker_port': broker_port,
'broker_virtual_host': broker_virtual_host,
'repository': repository,
}
ctx.invoke(setup, **setup_parameters)
def test_setup_fail_callback(self):
"""Make sure `determine_setup` works despite wrong initial values in case of correct callback"""
def correct_setup(interactive, dbinfo): # pylint: disable=unused-argument
return self.pg_test.dsn
postgres = Postgres(interactive=False, quiet=True, dbinfo={'port': '11111'}, determine_setup=False)
postgres.set_setup_fail_callback(correct_setup)
setup_success = postgres.determine_setup()
self.assertTrue(setup_success)
def quicksetup(
ctx, non_interactive, profile, email, first_name, last_name, institution, db_engine, db_backend, db_host, db_port,
db_name, db_username, db_password, su_db_name, su_db_username, su_db_password, repository
):
"""Setup a new profile in a fully automated fashion."""
# pylint: disable=too-many-arguments,too-many-locals
from aiida.manage.external.postgres import Postgres, manual_setup_instructions
dbinfo_su = {
'host': db_host,
'port': db_port,
'user': su_db_username,
'password': su_db_password,
}
postgres = Postgres(interactive=not non_interactive, quiet=False, dbinfo=dbinfo_su)
if not postgres.is_connected:
echo.echo_critical('failed to determine the PostgreSQL setup')
try:
create = True
if not postgres.dbuser_exists(db_username):
postgres.create_dbuser(db_username, db_password)
else:
db_name, create = postgres.check_db_name(db_name)
if create:
postgres.create_db(db_username, db_name)
except Exception as exception:
echo.echo_error(
'\n'.join([
'Oops! quicksetup was unable to create the AiiDA database for you.',
'For AiiDA to work, please either create the database yourself as follows:',
manual_setup_instructions(dbuser=su_db_username, dbname=su_db_name), '',
'Alternatively, give your (operating system) user permission to create postgresql databases' +
'and run quicksetup again.', ''
])
)
raise exception
# The contextual defaults or `verdi setup` are not being called when `invoking`, so we have to explicitly define
# them here, even though the `verdi setup` command would populate those when called from the command line.
setup_parameters = {
'non_interactive': non_interactive,
'profile': profile,
'email': email,
'first_name': first_name,
'last_name': last_name,
'institution': institution,
'db_engine': db_engine,
'db_backend': db_backend,
'db_name': db_name,
# from now on we connect as the AiiDA DB user, which may be forbidden when going via sockets
'db_host': db_host or 'localhost',
'db_port': db_port,
'db_username': db_username,
'db_password': db_password,
'repository': repository,
}
ctx.invoke(setup, **setup_parameters)
def create_aiida_db(self):
"""
Create the necessary database on the temporary postgres instance.
"""
if configuration.PROFILE is not None:
raise TestManagerError('AiiDA dbenv can not be loaded while creating a tests db environment')
if self.pg_cluster is None:
self.create_db_cluster()
self.postgres = Postgres(interactive=False, quiet=True, dbinfo=self.dbinfo)
# note: not using postgres.create_dbuser_db_safe here since we don't want prompts
self.postgres.create_dbuser(self.profile_info['database_username'], self.profile_info['database_password'])
self.postgres.create_db(self.profile_info['database_username'], self.profile_info['database_name'])
self.dbinfo = self.postgres.dbinfo
self.profile_info['database_hostname'] = self.postgres.host_for_psycopg2
self.profile_info['database_port'] = self.postgres.port_for_psycopg2
self._has_test_db = True
def test_determine_setup_fail(self):
"""Check that setup fails, if bad port is provided.
Note: In interactive mode, this would prompt for the connection details.
"""
postgres = Postgres(interactive=False, quiet=True, dbinfo={'port': '11111'})
self.assertFalse(postgres.is_connected)
def delete_db(profile, non_interactive=True, verbose=False):
"""
Delete an AiiDA database associated with an AiiDA profile.
:param profile: AiiDA Profile
:type profile: :class:`aiida.manage.configuration.profile.Profile`
:param non_interactive: do not prompt for configuration values, fail if not all values are given as kwargs.
:type non_interactive: bool
:param verbose: if True, print parameters of DB connection
:type verbose: bool
"""
from aiida.manage.configuration import get_config
from aiida.manage.external.postgres import Postgres
from aiida.common import json
postgres = Postgres.from_profile(profile,
interactive=not non_interactive,
quiet=False)
if verbose:
echo.echo_info('Parameters used to connect to postgres:')
echo.echo(json.dumps(postgres.dbinfo, indent=4))
database_name = profile.database_name
if not postgres.db_exists(database_name):
echo.echo_info(
"Associated database '{}' does not exist.".format(database_name))
elif non_interactive or click.confirm(
"Delete associated database '{}'?\n"
'WARNING: All data will be lost.'.format(database_name)):
echo.echo_info("Deleting database '{}'.".format(database_name))
postgres.drop_db(database_name)
user = profile.database_username
config = get_config()
users = [
available_profile.database_username
for available_profile in config.profiles
]
if not postgres.dbuser_exists(user):
echo.echo_info(
"Associated database user '{}' does not exist.".format(user))
elif users.count(user) > 1:
echo.echo_info(
"Associated database user '{}' is used by other profiles "
'and will not be deleted.'.format(user))
elif non_interactive or click.confirm(
"Delete database user '{}'?".format(user)):
echo.echo_info("Deleting user '{}'.".format(user))
postgres.drop_dbuser(user)
def create_aiida_db(self, pgtest=None):
"""
Create the necessary database on the temporary postgres instance.
By utilizing pgtest it is possible to forward initialization arguments to PGTest().
:param pgtest: a dictionary containing input to PGTest()
"""
if configuration.PROFILE is not None:
raise TestManagerError(
'AiiDA dbenv can not be loaded while creating a tests db environment'
)
if self.pg_cluster is None:
self.create_db_cluster(pgtest)
self.postgres = Postgres(interactive=False,
quiet=True,
dbinfo=self.dbinfo)
self.dbinfo = self.postgres.dbinfo.copy()
self.postgres.create_dbuser(self.profile_info['database_username'],
self.profile_info['database_password'])
self.postgres.create_db(self.profile_info['database_username'],
self.profile_info['database_name'])
self._has_test_db = True
def test_setup(self):
"""Test `verdi setup`."""
postgres = Postgres(interactive=False,
quiet=True,
dbinfo=self.pg_test.dsn)
postgres.determine_setup()
db_name = 'aiida_test_setup'
db_user = '******'
db_pass = '******'
postgres.create_dbuser(db_user, db_pass)
postgres.create_db(db_user, db_name)
configuration.reset_profile()
profile_name = 'testing'
user_email = '*****@*****.**'
user_first_name = 'John'
user_last_name = 'Smith'
user_institution = 'ECMA'
# Keep the `--profile` option last as a regression test for #2897 and #2907. Some of the other options have
# defaults, callbacks and or contextual defaults that might depend on it, but should not fail if they are parsed
# before the profile option is parsed.
options = [
'--non-interactive', '--email', user_email, '--first-name',
user_first_name, '--last-name', user_last_name, '--institution',
user_institution, '--db-name', db_name, '--db-username', db_user,
'--db-password', db_pass, '--db-port', self.pg_test.dsn['port'],
'--db-backend', self.backend, '--profile', profile_name
]
result = self.cli_runner.invoke(cmd_setup.setup, options)
self.assertClickResultNoException(result)
self.assertClickSuccess(result)
config = configuration.get_config()
self.assertIn(profile_name, config.profile_names)
profile = config.get_profile(profile_name)
profile.default_user = user_email
# Verify that the backend type of the created profile matches that of the profile for the current test session
self.assertEqual(self.backend, profile.database_backend)
user = orm.User.objects.get(email=user_email)
self.assertEqual(user.first_name, user_first_name)
self.assertEqual(user.last_name, user_last_name)
self.assertEqual(user.institution, user_institution)
def create_db(profile):
"""Create PostgreSQL database, if missing."""
dbinfo_su = {
'host': profile.database_hostname,
'port': profile.database_port,
'user': os.getenv("AIIDADB_SUPER_USER"),
'password': os.getenv("AIIDADB_SUPER_PASS"),
'database': 'template1',
}
postgres = Postgres(interactive=False, quiet=False,
dbinfo=dbinfo_su) #, determine_setup=False)
if not postgres.is_connected:
raise ConnectionError("Unable to connect as super user")
try:
if not postgres.dbuser_exists(dbuser=profile.database_username):
postgres.create_dbuser(dbuser=profile.database_username,
dbpass=profile.database_password)
if not postgres.db_exists(dbname=profile.database_name):
postgres.create_db(profile.database_username,
profile.database_name)
# Fill DB with vanilla content
load_profile(profile.name)
backend = get_manager()._load_backend(schema_check=False) # pylint: disable=protected-access
try:
backend.migrate()
except Exception as exception: # pylint: disable=broad-except
print(
'database migration failed, probably because connection details are incorrect:\n{}'
.format(exception))
# Create the user if it does not yet exist
created, user = orm.User.objects.get_or_create(
email=profile.default_user,
first_name='AiiDA',
last_name='EXPLORER',
institution='WEBSERVICE')
if created:
user.store()
profile.default_user = user.email
except:
print(traceback.format_exc())
class TemporaryProfileManager(ProfileManager):
"""
Manage the life cycle of a completely separated and temporary AiiDA environment.
* No profile / database setup required
* Tests run via the TemporaryProfileManager never pollute the user's working environment
Filesystem:
* temporary ``.aiida`` configuration folder
* temporary repository folder
Database:
* temporary database cluster (via the ``pgtest`` package)
* with ``aiida`` database user
* with ``aiida_db`` database
AiiDA:
* configured to use the temporary configuration
* sets up a temporary profile for tests
All of this happens automatically when using the corresponding tests classes & tests runners (unittest)
or fixtures (pytest).
Example::
tests = TemporaryProfileManager(backend=backend)
tests.create_aiida_db() # set up only the database
tests.create_profile() # set up a profile (creates the db too if necessary)
# ready for tests
# run tests 1
tests.reset_db()
# database ready for independent tests 2
# run tests 2
tests.destroy_all()
# everything cleaned up
"""
_test_case = None
def __init__(self, backend=BACKEND_DJANGO, pgtest=None): # pylint: disable=super-init-not-called
"""Construct a TemporaryProfileManager
:param backend: a database backend
:param pgtest: a dictionary of arguments to be passed to PGTest() for starting the postgresql cluster,
e.g. {'pg_ctl': '/somepath/pg_ctl'}. Should usually not be necessary.
"""
from aiida.manage.configuration import settings
self.dbinfo = {}
self.profile_info = _DEFAULT_PROFILE_INFO
self.profile_info['database_backend'] = backend
self._pgtest = pgtest or {}
self.pg_cluster = None
self.postgres = None
self._profile = None
self._has_test_db = False
self._backup = {}
self._backup['config'] = configuration.CONFIG
self._backup['config_dir'] = settings.AIIDA_CONFIG_FOLDER
self._backup['profile'] = configuration.PROFILE
@property
def profile_dictionary(self):
"""Profile parameters.
Used to set up AiiDA profile from self.profile_info dictionary.
"""
dictionary = {
'database_engine': self.profile_info['database_engine'],
'database_backend': self.profile_info['database_backend'],
'database_port': self.dbinfo.get('port'),
'database_hostname': self.dbinfo.get('host'),
'database_name': self.profile_info.get('database_name'),
'database_username': self.profile_info.get('database_username'),
'database_password': self.profile_info.get('database_password'),
'repository_uri': 'file://' + self.repo,
}
return dictionary
def create_db_cluster(self):
"""
Create the database cluster using PGTest.
"""
from pgtest.pgtest import PGTest
if self.pg_cluster is not None:
raise TestManagerError(
'Running temporary postgresql cluster detected.' +
'Use destroy_all() before creating a new cluster.')
self.pg_cluster = PGTest(**self._pgtest)
self.dbinfo.update(self.pg_cluster.dsn)
def create_aiida_db(self):
"""
Create the necessary database on the temporary postgres instance.
"""
if configuration.PROFILE is not None:
raise TestManagerError(
'AiiDA dbenv can not be loaded while creating a tests db environment'
)
if self.pg_cluster is None:
self.create_db_cluster()
self.postgres = Postgres(interactive=False,
quiet=True,
dbinfo=self.dbinfo)
self.dbinfo = self.postgres.dbinfo.copy()
self.postgres.create_dbuser(self.profile_info['database_username'],
self.profile_info['database_password'])
self.postgres.create_db(self.profile_info['database_username'],
self.profile_info['database_name'])
self._has_test_db = True
def create_profile(self):
"""
Set AiiDA to use the tests config dir and create a default profile there
Warning: the AiiDA dbenv must not be loaded when this is called!
"""
from aiida.manage.configuration import settings, load_profile, Profile
if not self._has_test_db:
self.create_aiida_db()
if not self.root_dir:
self.root_dir = tempfile.mkdtemp()
configuration.CONFIG = None
settings.AIIDA_CONFIG_FOLDER = self.config_dir
configuration.PROFILE = None
create_instance_directories()
profile_name = self.profile_info['name']
config = configuration.get_config(create=True)
profile = Profile(profile_name, self.profile_dictionary)
config.add_profile(profile)
config.set_default_profile(profile_name).store()
self._profile = profile
load_profile(profile_name)
backend = manager.get_manager()._load_backend(schema_check=False)
backend.migrate()
self._select_db_test_case(backend=self._profile.database_backend)
self.init_db()
def repo_ok(self):
return bool(self.repo and os.path.isdir(os.path.dirname(self.repo)))
@property
def repo(self):
return self._return_dir(self.profile_info['repo_dir'])
def _return_dir(self, dir_path):
"""Return a path to a directory from the fs environment"""
if os.path.isabs(dir_path):
return dir_path
return os.path.join(self.root_dir, dir_path)
@property
def backend(self):
return self.profile_info['backend']
@backend.setter
def backend(self, backend):
if self.has_profile_open():
raise TestManagerError(
'backend cannot be changed after setting up the environment')
valid_backends = [BACKEND_DJANGO, BACKEND_SQLA]
if backend not in valid_backends:
raise ValueError('invalid backend {}, must be one of {}'.format(
backend, valid_backends))
self.profile_info['backend'] = backend
@property
def config_dir_ok(self):
return bool(self.config_dir and os.path.isdir(self.config_dir))
@property
def config_dir(self):
return self._return_dir(self.profile_info['config_dir'])
@property
def root_dir(self):
return self.profile_info['root_path']
@root_dir.setter
def root_dir(self, root_dir):
self.profile_info['root_path'] = root_dir
@property
def root_dir_ok(self):
return bool(self.root_dir and os.path.isdir(self.root_dir))
def destroy_all(self):
"""Remove all traces of the tests run"""
from aiida.manage.configuration import settings
if self.root_dir:
shutil.rmtree(self.root_dir)
self.root_dir = None
if self.pg_cluster:
self.pg_cluster.close()
self.pg_cluster = None
self._has_test_db = False
self._profile = None
self._user = None
if 'config' in self._backup:
configuration.CONFIG = self._backup['config']
if 'config_dir' in self._backup:
settings.AIIDA_CONFIG_FOLDER = self._backup['config_dir']
if 'profile' in self._backup:
configuration.PROFILE = self._backup['profile']
def has_profile_open(self):
return self._profile is not None
def _setup_postgres(self):
return Postgres(interactive=False, quiet=True, dbinfo=self.pg_test.dsn)
def test_determine_setup_fail(self):
postgres = Postgres(interactive=False, quiet=True, dbinfo={'port': '11111'})
self.assertFalse(postgres.is_connected)