class PostgresTest(unittest.TestCase):
"""Test the public API provided by the `Postgres` class"""
def setUp(self):
"""Set up a temporary database cluster for testing potentially destructive operations"""
self.pg_test = PGTest()
self.postgres = Postgres(port=self.pg_test.port,
interactive=False,
quiet=True)
self.dbuser = '******'
self.dbpass = '******'
self.dbname = 'aiida_db'
def _setup_postgres(self):
self.postgres.dbinfo = self.pg_test.dsn
self.postgres.determine_setup()
def test_determine_setup_fail(self):
self.postgres.set_port('11111')
setup_success = self.postgres.determine_setup()
self.assertFalse(setup_success)
def test_determine_setup_success(self):
self._setup_postgres()
self.assertTrue(self.postgres.pg_execute)
def test_setup_fail_callback(self):
"""Make sure `determine_setup` works despite wrong initial values in case of correct callback"""
def correct_setup(interactive, dbinfo): # pylint: disable=unused-argument
return self.pg_test.dsn
self.postgres.set_port(11111)
self.postgres.set_setup_fail_callback(correct_setup)
self.postgres.determine_setup()
self.assertTrue(self.postgres.pg_execute)
@mock.patch('aiida.control.postgres._try_connect',
new=_try_connect_always_fail)
@mock.patch('aiida.control.postgres._try_subcmd')
def test_fallback_on_subcmd(self, try_subcmd):
"""Ensure that accessing postgres via subcommand is tried if psychopg does not work."""
self._setup_postgres()
self.assertTrue(try_subcmd.call_count >= 1)
def test_create_drop_db_user(self):
"""Check creating and dropping a user works"""
self._setup_postgres()
self.postgres.create_dbuser(self.dbuser, self.dbpass)
self.assertTrue(self.postgres.dbuser_exists(self.dbuser))
self.postgres.drop_dbuser(self.dbuser)
self.assertFalse(self.postgres.dbuser_exists(self.dbuser))
def test_create_drop_db(self):
"""Check creating & destroying a database"""
self._setup_postgres()
self.postgres.create_dbuser(self.dbuser, self.dbpass)
self.postgres.create_db(self.dbuser, self.dbname)
self.assertTrue(self.postgres.db_exists(self.dbname))
self.postgres.drop_db(self.dbname)
self.assertFalse(self.postgres.db_exists(self.dbname))
def quicksetup(self, profile, email, first_name, last_name, institution,
backend, db_port, db_user, db_user_pw, db_name, repo,
set_default, non_interactive):
'''Set up a sane aiida configuration with as little interaction as possible.'''
from aiida.common.setup import create_base_dirs, AIIDA_CONFIG_FOLDER
create_base_dirs()
aiida_dir = os.path.expanduser(AIIDA_CONFIG_FOLDER)
# access postgres
postgres = Postgres(port=db_port,
interactive=bool(not non_interactive),
quiet=False)
postgres.set_setup_fail_callback(prompt_db_info)
success = postgres.determine_setup()
if not success:
sys.exit(1)
# default database name is <profile>_<login-name>
# this ensures that for profiles named test_... the database will also
# be named test_...
import getpass
osuser = getpass.getuser()
dbname = db_name or profile + '_' + osuser
# default database user name is aiida_qs_<login-name>
# default password is random
dbuser = db_user or 'aiida_qs_' + osuser
from aiida.common.setup import generate_random_secret_key
dbpass = db_user_pw or generate_random_secret_key()
# check if there is a profile that contains the db user already
# and if yes, take the db user password from there
# This is ok because a user can only see his own config files
from aiida.common.setup import (set_default_profile, get_or_create_config)
confs = get_or_create_config()
profs = confs.get('profiles', {})
for v in profs.itervalues():
if v.get('AIIDADB_USER', '') == dbuser and not db_user_pw:
dbpass = v.get('AIIDADB_PASS')
print 'using found password for {}'.format(dbuser)
break
try:
create = True
if not postgres.dbuser_exists(dbuser):
postgres.create_dbuser(dbuser, dbpass)
else:
dbname, create = _check_db_name(dbname, postgres)
if create:
postgres.create_db(dbuser, dbname)
except Exception as e:
click.echo('\n'.join([
'Oops! Something went wrong while creating the database for you.',
'You may continue with the quicksetup, however:',
'For aiida to work correctly you will have to do that yourself as follows.',
manual_setup_instructions(dbuser=dbuser, dbname=dbname), '',
'Or setup your (OS-level) user to have permissions to create databases and rerun quicksetup.',
''
]))
raise e
# create a profile, by default 'quicksetup' and prompt the user if
# already exists
confs = get_or_create_config()
profile_name = profile or 'quicksetup'
write_profile = False
while not write_profile:
if profile_name in confs.get('profiles', {}):
if click.confirm(
'overwrite existing profile {}?'.format(profile_name)):
write_profile = True
else:
profile_name = click.prompt('new profile name', type=str)
else:
write_profile = True
dbhost = postgres.dbinfo.get('host', 'localhost')
dbport = postgres.dbinfo.get('port', '5432')
from os.path import isabs
repo = repo or 'repository-{}/'.format(profile_name)
if not isabs(repo):
repo = os.path.join(aiida_dir, repo)
setup_args = {
'backend': backend,
'email': email,
'db_host': dbhost,
'db_port': dbport,
'db_name': dbname,
'db_user': dbuser,
'db_pass': dbpass,
'repo': repo,
'first_name': first_name,
'last_name': last_name,
'institution': institution,
'force_overwrite': write_profile,
}
setup(profile_name, only_config=False, non_interactive=True, **setup_args)
# Loop over all valid processes and check if a default profile is set for them
# If not set the newly created profile as default, otherwise prompt whether to override
from aiida.cmdline.commands.profile import valid_processes
default_profiles = confs.get('default_profiles', {})
for process in valid_processes:
# if the user specifies whether to override that's fine
if set_default in [True, False]:
_set_default = set_default
# otherwise we may need to ask
else:
default_profile = default_profiles.get(process, '')
if default_profile:
_set_default = click.confirm(
"The default profile for the '{}' process is set to '{}': "
"do you want to set the newly created '{}' as the new default? (can be reverted later)"
.format(process, default_profile, profile_name))
# if there are no other default profiles, we don't need to ask
else:
_set_default = True
if _set_default:
set_default_profile(process, profile_name, force_rewrite=True)
def quicksetup(profile_name, only_config, set_default, non_interactive, backend, db_host, db_port, db_name, db_username,
db_password, repository, email, first_name, last_name, institution):
"""Set up a sane configuration with as little interaction as possible."""
from aiida.common.setup import create_base_dirs, AIIDA_CONFIG_FOLDER
create_base_dirs()
aiida_dir = os.path.expanduser(AIIDA_CONFIG_FOLDER)
# access postgres
postgres = Postgres(host=db_host, port=db_port, interactive=bool(not non_interactive), quiet=False)
postgres.set_setup_fail_callback(prompt_db_info)
success = postgres.determine_setup()
if not success:
sys.exit(1)
# default database name is <profile_name>_<login-name>
# this ensures that for profiles named test_... the database will also
# be named test_...
import getpass
osuser = getpass.getuser()
dbname = db_name or profile_name + '_' + osuser
# default database user name is aiida_qs_<login-name>
# default password is random
dbuser = db_username or 'aiida_qs_' + osuser
from aiida.common.setup import generate_random_secret_key
dbpass = db_password or generate_random_secret_key()
# check if there is a profile that contains the db user already
# and if yes, take the db user password from there
# This is ok because a user can only see his own config files
from aiida.common.setup import get_or_create_config
confs = get_or_create_config()
profs = confs.get('profiles', {})
for profile in profs.itervalues():
if profile.get('AIIDADB_USER', '') == dbuser and not db_password:
dbpass = profile.get('AIIDADB_PASS')
print 'using found password for {}'.format(dbuser)
break
try:
create = True
if not postgres.dbuser_exists(dbuser):
postgres.create_dbuser(dbuser, dbpass)
else:
dbname, create = _check_db_name(dbname, postgres)
if create:
postgres.create_db(dbuser, dbname)
except Exception as exception:
click.echo('\n'.join([
'Oops! Something went wrong while creating the database for you.',
'You may continue with the quicksetup, however:',
'For aiida to work correctly you will have to do that yourself as follows.',
manual_setup_instructions(dbuser=dbuser, dbname=dbname), '',
'Or setup your (OS-level) user to have permissions to create databases and rerun quicksetup.', ''
]))
raise exception
# create a profile, by default 'quicksetup' and prompt the user if
# already exists
confs = get_or_create_config()
profile_name = profile_name or 'quicksetup'
write_profile = False
while not write_profile:
if profile_name in confs.get('profiles', {}):
if click.confirm('overwrite existing profile {}?'.format(profile_name)):
write_profile = True
else:
profile_name = click.prompt('new profile name', type=str)
else:
write_profile = True
dbhost = postgres.dbinfo.get('host', 'localhost')
dbport = postgres.dbinfo.get('port', '5432')
from os.path import isabs
repo = repository or 'repository-{}/'.format(profile_name)
if not isabs(repo):
repo = os.path.join(aiida_dir, repo)
setup_args = {
'backend': backend,
'email': email,
'db_host': dbhost,
'db_port': dbport,
'db_name': dbname,
'db_user': dbuser,
'db_pass': dbpass,
'repo': repo,
'first_name': first_name,
'last_name': last_name,
'institution': institution,
'force_overwrite': write_profile,
}
setup_profile(profile_name, only_config=only_config, set_default=set_default, non_interactive=True, **setup_args)
class FixtureManager(object):
"""
Manage the life cycle of a completely separated and temporary AiiDA environment
* No previously created database of profile is required to run tests using this
environment
* Tests using this environment will never pollute the user's work environment
Example::
fixtures = FixtureManager()
fixtures.create_aiida_db() # set up only the database
fixtures.create_profile() # set up a profile (creates the db too if necessary)
# ready for testing
# run test 1
fixtures.reset_db()
# database ready for independent test 2
# run test 2
fixtures.destroy_all()
# everything cleaned up
Usage (unittest): See the :py:class:`PluginTestCase` and the :py:class:`TestRunner`.
Usage (pytest)::
import pytest
@pytest.fixture(scope='session')
def aiida_profile():
with aiida.utils.fixtures.fixture_manager() as fixture_mgr:
fixture_mgr.create_profile()
yield fixture_manager
@pytest.fixture(scope='function')
def test_data(aiida_profile):
# load my test data
yield
fixture_manager.reset_db()
def test_my_stuff(test_data):
# run a test
"""
def __init__(self):
self.db_params = {}
self.fs_env = {'repo': 'test_repo', 'config': '.aiida'}
self.profile_info = {
'backend': 'django',
'email': '*****@*****.**',
'first_name': 'AiiDA',
'last_name': 'Plugintest',
'institution': 'aiidateam',
'db_user': '******',
'db_pass': '******',
'db_name': 'aiida_db'
}
self.pg_cluster = None
self.postgres = None
self.__is_running_on_test_db = False
self.__is_running_on_test_profile = False
self._backup = {}
self._backup['config_dir'] = aiida_cfg.AIIDA_CONFIG_FOLDER
self._backup['profile'] = backend_settings.AIIDADB_PROFILE
def create_db_cluster(self):
if not self.pg_cluster:
self.pg_cluster = PGTest(max_connections=256)
self.db_params.update(self.pg_cluster.dsn)
def create_aiida_db(self):
"""Create the necessary database on the temporary postgres instance"""
if is_dbenv_loaded():
raise FixtureError(
'AiiDA dbenv can not be loaded while creating a test db environment'
)
if not self.db_params:
self.create_db_cluster()
self.postgres = Postgres(interactive=False, quiet=True)
self.postgres.dbinfo = self.db_params
self.postgres.determine_setup()
self.db_params = self.postgres.dbinfo
if not self.postgres.pg_execute:
raise FixtureError(
'Could not connect to the test postgres instance')
self.postgres.create_dbuser(self.db_user, self.db_pass)
self.postgres.create_db(self.db_user, self.db_name)
self.__is_running_on_test_db = True
def create_root_dir(self):
self.root_dir = tempfile.mkdtemp()
def create_profile(self):
"""
Set AiiDA to use the test config dir and create a default profile there
Warning: the AiiDA dbenv must not be loaded when this is called!
"""
if is_dbenv_loaded():
raise FixtureError(
'AiiDA dbenv can not be loaded while creating a test profile')
if not self.__is_running_on_test_db:
self.create_aiida_db()
from aiida.cmdline.verdilib import setup
if not self.root_dir:
self.create_root_dir()
print(self.root_dir, self.config_dir)
aiida_cfg.AIIDA_CONFIG_FOLDER = self.config_dir
backend_settings.AIIDADB_PROFILE = None
aiida_cfg.create_base_dirs()
profile_name = 'test_profile'
setup(profile=profile_name,
only_config=False,
non_interactive=True,
**self.profile)
aiida_cfg.set_default_profile('verdi', profile_name)
aiida_cfg.set_default_profile('daemon', profile_name)
self.__is_running_on_test_profile = True
def reset_db(self):
"""Cleans all data from the database between tests"""
if not self.__is_running_on_test_profile:
raise FixtureError(
'No test profile has been set up yet, can not reset the db')
if self.profile_info['backend'] == BACKEND_DJANGO:
self.__clean_db_django()
elif self.profile_info['backend'] == BACKEND_SQLA:
self.__clean_db_sqla()
@property
def profile(self):
"""Profile parameters"""
profile = {
'backend': self.backend,
'email': self.email,
'repo': self.repo,
'db_host': self.db_host,
'db_port': self.db_port,
'db_user': self.db_user,
'db_pass': self.db_pass,
'db_name': self.db_name,
'first_name': self.first_name,
'last_name': self.last_name,
'institution': self.institution
}
return profile
@property
def db_host(self):
return self.db_params.get('host')
@db_host.setter
def db_host(self, hostname):
self.db_params['host'] = hostname
@property
def first_name(self):
return self.profile_info['first_name']
@first_name.setter
def first_name(self, name):
self.profile_info['first_name'] = name
@property
def last_name(self):
return self.profile_info['last_name']
@last_name.setter
def last_name(self, name):
self.profile_info['last_name'] = name
@property
def institution(self):
return self.profile_info['institution']
@institution.setter
def institution(self, institution):
self.profile_info['institution'] = institution
@property
def db_port(self):
return self.db_params.get('port', None)
@db_port.setter
def db_port(self, port):
self.db_params['port'] = str(port)
def repo_ok(self):
return bool(self.repo and path.isdir(path.dirname(self.repo)))
@property
def repo(self):
return self._return_dir('repo')
@repo.setter
def repo(self, repo_dir):
self.fs_env['repo'] = repo_dir
def _return_dir(self, key):
"""Return a path to a directory from the fs environment"""
dir_path = self.fs_env[key]
if not dir_path:
raise FixtureError('no directory set for {}'.format(key))
elif path.isabs(dir_path):
return dir_path
return path.join(self.root_dir, dir_path)
@property
def email(self):
return self.profile_info['email']
@email.setter
def email(self, email):
self.profile_info['email'] = email
@property
def backend(self):
return self.profile_info['backend']
@backend.setter
def backend(self, backend):
valid_backends = [BACKEND_DJANGO, BACKEND_SQLA]
if backend not in valid_backends:
raise ValueError('invalid backend {}, must be one of {}'.format(
backend, valid_backends))
self.profile_info['backend'] = backend
@property
def config_dir_ok(self):
return bool(self.config_dir and path.isdir(self.config_dir))
@property
def config_dir(self):
return self._return_dir('config')
@config_dir.setter
def config_dir(self, config_dir):
self.fs_env['config'] = config_dir
@property
def db_user(self):
return self.profile_info['db_user']
@db_user.setter
def db_user(self, user):
self.profile_info['db_user'] = user
@property
def db_pass(self):
return self.profile_info['db_pass']
@db_pass.setter
def db_pass(self, passwd):
self.profile_info['db_pass'] = passwd
@property
def db_name(self):
return self.profile_info['db_name']
@db_name.setter
def db_name(self, name):
self.profile_info['db_name'] = name
@property
def root_dir(self):
return self.fs_env.get('root', '')
@root_dir.setter
def root_dir(self, root_dir):
self.fs_env['root'] = root_dir
@property
def root_dir_ok(self):
return bool(self.root_dir and path.isdir(self.root_dir))
def destroy_all(self):
"""Remove all traces of the test run"""
if self.root_dir:
shutil.rmtree(self.root_dir)
self.root_dir = None
if self.pg_cluster:
self.pg_cluster.close()
self.pg_cluster = None
self.__is_running_on_test_db = False
self.__is_running_on_test_profile = False
if 'config_dir' in self._backup:
aiida_cfg.AIIDA_CONFIG_FOLDER = self._backup['config_dir']
if 'profile' in self._backup:
backend_settings.AIIDADB_PROFILE = self._backup['profile']
@staticmethod
def __clean_db_django():
from aiida.backends.djsite.db.testbase import DjangoTests
DjangoTests().clean_db()
def __clean_db_sqla(self):
"""Clean database for sqlalchemy backend"""
from aiida.backends.sqlalchemy.tests.testbase import SqlAlchemyTests
from aiida.backends.sqlalchemy import get_scoped_session
from aiida.orm import User
user = User.search_for_users(email=self.email)[0]
new_user = User(email=user.email)
new_user.first_name = user.first_name
new_user.last_name = user.last_name
new_user.institution = user.institution
sqla_testcase = SqlAlchemyTests()
sqla_testcase.test_session = get_scoped_session()
sqla_testcase.clean_db()
# that deleted our user, we need to recreate it
new_user.force_save()
def has_profile_open(self):
return self.__is_running_on_test_profile
class SetupTestCase(unittest.TestCase):
"""Test ``verdi setup``"""
def setUp(self):
self.runner = CliRunner()
backend = os.environ.get('TEST_AIIDA_BACKEND', 'django')
self.backend = 'django' if backend == 'django' else 'sqlalchemy'
self.pg_test = PGTest()
self.postgres = Postgres(port=self.pg_test.port, interactive=False, quiet=True)
self.postgres.dbinfo = self.pg_test.dsn
self.postgres.determine_setup()
self.dbuser = '******'
self.dbpass = '******'
self.dbname = 'aiida_test_setup'
self.postgres.create_dbuser(self.dbuser, self.dbpass)
self.postgres.create_db(self.dbuser, self.dbname)
self.repo = abspath('./aiida_radames')
def tearDown(self):
self.postgres.drop_db(self.dbname)
self.postgres.drop_dbuser(self.dbuser)
self.pg_test.close()
def test_user_setup(self):
backend_settings.AIIDADB_PROFILE = None
result = self.runner.invoke(
_setup_cmd, [
'radames',
'--non-interactive',
'--backend={}'.format(self.backend),
'[email protected]',
'--first-name=Radames',
'--last-name=Verdi',
'--institution=Scala',
'--repo={}'.format(self.repo),
'--db_host=localhost',
'--db_port={}'.format(self.pg_test.port),
'--db_name={}'.format(self.dbname),
'--db_user={}'.format(self.dbuser),
'--db_pass={}'.format(self.dbpass),
'--no-password'])
self.assertFalse(result.exception, msg=get_debug_msg(result))
def test_user_configure(self):
backend_settings.AIIDADB_PROFILE = None
self.runner.invoke(
_setup_cmd, [
'radames2',
'--non-interactive',
'--backend={}'.format(self.backend),
'[email protected]',
'--first-name=Radames',
'--last-name=Verdi',
'--institution=Scala',
'--repo={}'.format(self.repo),
'--db_host=localhost',
'--db_port={}'.format(self.pg_test.port),
'--db_name={}'.format(self.dbname),
'--db_user={}'.format(self.dbuser),
'--db_pass={}'.format(self.dbpass),
'--no-password'])
backend_settings.AIIDADB_PROFILE = None
result = self.runner.invoke(
_setup_cmd,
['radames2', '--only-config'],
input='yes\[email protected]\npostgresql_psycopg2\n\n\n\n\n\n{repo}\nRadames2\nVerdi2\nScala2\nyes\nno\n'.format(
repo=self.repo
),
catch_exceptions=False
)
self.assertFalse(result.exception, msg=get_debug_msg(result))
class SetupTestCase(unittest.TestCase):
"""Test `verdi setup`."""
def setUp(self):
self.runner = CliRunner()
backend = os.environ.get('TEST_AIIDA_BACKEND', 'django')
self.backend = 'django' if backend == 'django' else 'sqlalchemy'
self.pg_test = PGTest()
self.postgres = Postgres(port=self.pg_test.port,
interactive=False,
quiet=True)
self.postgres.dbinfo = self.pg_test.dsn
self.postgres.determine_setup()
self.dbuser = '******'
self.dbpass = '******'
self.dbname = 'aiida_test_setup_{}'.format(self.backend)
self.postgres.create_dbuser(self.dbuser, self.dbpass)
self.postgres.create_db(self.dbuser, self.dbname)
self.repository = abspath('./aiida_radames_{}'.format(self.backend))
def tearDown(self):
self.postgres.drop_db(self.dbname)
self.postgres.drop_dbuser(self.dbuser)
self.pg_test.close()
def test_user_setup(self):
"""
Test `verdi setup` non-interactively
"""
backend_settings.AIIDADB_PROFILE = None
result = self.runner.invoke(setup, [
'--non-interactive', '--backend={}'.format(
self.backend), '[email protected]',
'--first-name=Radames', '--last-name=Verdi', '--institution=Scala',
'--repository={}'.format(self.repository), '--db-host=localhost',
'--db-port={}'.format(self.pg_test.port), '--db-name={}'.format(
self.dbname), '--db-username={}'.format(
self.dbuser), '--db-password={}'.format(
self.dbpass), 'radames_{}'.format(self.backend)
])
self.assertFalse(result.exception, msg=get_debug_msg(result))
def test_user_configure(self):
"""
Test `verdi setup` configure user
"""
backend_settings.AIIDADB_PROFILE = None
self.runner.invoke(setup, [
'--non-interactive', '--backend={}'.format(
self.backend), '[email protected]',
'--first-name=Radames', '--last-name=Verdi', '--institution=Scala',
'--repository={}'.format(self.repository), '--db-host=localhost',
'--db-port={}'.format(self.pg_test.port), '--db-name={}'.format(
self.dbname), '--db-username={}'.format(
self.dbuser), '--db-password={}'.format(
self.dbpass), 'radames2_{}'.format(self.backend)
])
tpl = '{email}\n{first_name}\n{last_name}\n{institution}\nyes\n{email}\n{engine}\n\n\n\n\n\n{repo}\nno\n\n'
backend_settings.AIIDADB_PROFILE = None
result = self.runner.invoke(
setup, ['radames2_{}'.format(self.backend), '--only-config'],
input=tpl.format(email='*****@*****.**',
first_name='Radames2',
last_name='Verdi2',
institution='Scala2',
engine='postgresql_psycopg2',
repo=self.repository),
catch_exceptions=False)
self.assertFalse(result.exception, msg=get_debug_msg(result))