Esempio n. 1
0
class PostgresTest(unittest.TestCase):
    """Test the public API provided by the `Postgres` class"""
    def setUp(self):
        """Set up a temporary database cluster for testing potentially destructive operations"""
        self.pg_test = PGTest()
        self.postgres = Postgres(port=self.pg_test.port,
                                 interactive=False,
                                 quiet=True)
        self.dbuser = '******'
        self.dbpass = '******'
        self.dbname = 'aiida_db'

    def _setup_postgres(self):
        self.postgres.dbinfo = self.pg_test.dsn
        self.postgres.determine_setup()

    def test_determine_setup_fail(self):
        self.postgres.set_port('11111')
        setup_success = self.postgres.determine_setup()
        self.assertFalse(setup_success)

    def test_determine_setup_success(self):
        self._setup_postgres()
        self.assertTrue(self.postgres.pg_execute)

    def test_setup_fail_callback(self):
        """Make sure `determine_setup` works despite wrong initial values in case of correct callback"""
        def correct_setup(interactive, dbinfo):  # pylint: disable=unused-argument
            return self.pg_test.dsn

        self.postgres.set_port(11111)
        self.postgres.set_setup_fail_callback(correct_setup)
        self.postgres.determine_setup()
        self.assertTrue(self.postgres.pg_execute)

    @mock.patch('aiida.control.postgres._try_connect',
                new=_try_connect_always_fail)
    @mock.patch('aiida.control.postgres._try_subcmd')
    def test_fallback_on_subcmd(self, try_subcmd):
        """Ensure that accessing postgres via subcommand is tried if psychopg does not work."""
        self._setup_postgres()
        self.assertTrue(try_subcmd.call_count >= 1)

    def test_create_drop_db_user(self):
        """Check creating and dropping a user works"""
        self._setup_postgres()
        self.postgres.create_dbuser(self.dbuser, self.dbpass)
        self.assertTrue(self.postgres.dbuser_exists(self.dbuser))
        self.postgres.drop_dbuser(self.dbuser)
        self.assertFalse(self.postgres.dbuser_exists(self.dbuser))

    def test_create_drop_db(self):
        """Check creating & destroying a database"""
        self._setup_postgres()
        self.postgres.create_dbuser(self.dbuser, self.dbpass)
        self.postgres.create_db(self.dbuser, self.dbname)
        self.assertTrue(self.postgres.db_exists(self.dbname))
        self.postgres.drop_db(self.dbname)
        self.assertFalse(self.postgres.db_exists(self.dbname))
Esempio n. 2
0
def quicksetup(self, profile, email, first_name, last_name, institution,
               backend, db_port, db_user, db_user_pw, db_name, repo,
               set_default, non_interactive):
    '''Set up a sane aiida configuration with as little interaction as possible.'''
    from aiida.common.setup import create_base_dirs, AIIDA_CONFIG_FOLDER
    create_base_dirs()

    aiida_dir = os.path.expanduser(AIIDA_CONFIG_FOLDER)

    # access postgres
    postgres = Postgres(port=db_port,
                        interactive=bool(not non_interactive),
                        quiet=False)
    postgres.set_setup_fail_callback(prompt_db_info)
    success = postgres.determine_setup()
    if not success:
        sys.exit(1)

    # default database name is <profile>_<login-name>
    # this ensures that for profiles named test_... the database will also
    # be named test_...
    import getpass
    osuser = getpass.getuser()
    dbname = db_name or profile + '_' + osuser

    # default database user name is aiida_qs_<login-name>
    # default password is random
    dbuser = db_user or 'aiida_qs_' + osuser
    from aiida.common.setup import generate_random_secret_key
    dbpass = db_user_pw or generate_random_secret_key()

    # check if there is a profile that contains the db user already
    # and if yes, take the db user password from there
    # This is ok because a user can only see his own config files
    from aiida.common.setup import (set_default_profile, get_or_create_config)
    confs = get_or_create_config()
    profs = confs.get('profiles', {})
    for v in profs.itervalues():
        if v.get('AIIDADB_USER', '') == dbuser and not db_user_pw:
            dbpass = v.get('AIIDADB_PASS')
            print 'using found password for {}'.format(dbuser)
            break

    try:
        create = True
        if not postgres.dbuser_exists(dbuser):
            postgres.create_dbuser(dbuser, dbpass)
        else:
            dbname, create = _check_db_name(dbname, postgres)
        if create:
            postgres.create_db(dbuser, dbname)
    except Exception as e:
        click.echo('\n'.join([
            'Oops! Something went wrong while creating the database for you.',
            'You may continue with the quicksetup, however:',
            'For aiida to work correctly you will have to do that yourself as follows.',
            manual_setup_instructions(dbuser=dbuser, dbname=dbname), '',
            'Or setup your (OS-level) user to have permissions to create databases and rerun quicksetup.',
            ''
        ]))
        raise e

    # create a profile, by default 'quicksetup' and prompt the user if
    # already exists
    confs = get_or_create_config()
    profile_name = profile or 'quicksetup'
    write_profile = False
    while not write_profile:
        if profile_name in confs.get('profiles', {}):
            if click.confirm(
                    'overwrite existing profile {}?'.format(profile_name)):
                write_profile = True
            else:
                profile_name = click.prompt('new profile name', type=str)
        else:
            write_profile = True

    dbhost = postgres.dbinfo.get('host', 'localhost')
    dbport = postgres.dbinfo.get('port', '5432')

    from os.path import isabs
    repo = repo or 'repository-{}/'.format(profile_name)
    if not isabs(repo):
        repo = os.path.join(aiida_dir, repo)

    setup_args = {
        'backend': backend,
        'email': email,
        'db_host': dbhost,
        'db_port': dbport,
        'db_name': dbname,
        'db_user': dbuser,
        'db_pass': dbpass,
        'repo': repo,
        'first_name': first_name,
        'last_name': last_name,
        'institution': institution,
        'force_overwrite': write_profile,
    }
    setup(profile_name, only_config=False, non_interactive=True, **setup_args)

    # Loop over all valid processes and check if a default profile is set for them
    # If not set the newly created profile as default, otherwise prompt whether to override
    from aiida.cmdline.commands.profile import valid_processes

    default_profiles = confs.get('default_profiles', {})

    for process in valid_processes:

        # if the user specifies whether to override that's fine
        if set_default in [True, False]:
            _set_default = set_default
        # otherwise we may need to ask
        else:
            default_profile = default_profiles.get(process, '')
            if default_profile:
                _set_default = click.confirm(
                    "The default profile for the '{}' process is set to '{}': "
                    "do you want to set the newly created '{}' as the new default? (can be reverted later)"
                    .format(process, default_profile, profile_name))
            # if there are no other default profiles, we don't need to ask
            else:
                _set_default = True

        if _set_default:
            set_default_profile(process, profile_name, force_rewrite=True)
Esempio n. 3
0
def quicksetup(profile_name, only_config, set_default, non_interactive, backend, db_host, db_port, db_name, db_username,
               db_password, repository, email, first_name, last_name, institution):
    """Set up a sane configuration with as little interaction as possible."""
    from aiida.common.setup import create_base_dirs, AIIDA_CONFIG_FOLDER
    create_base_dirs()

    aiida_dir = os.path.expanduser(AIIDA_CONFIG_FOLDER)

    # access postgres
    postgres = Postgres(host=db_host, port=db_port, interactive=bool(not non_interactive), quiet=False)
    postgres.set_setup_fail_callback(prompt_db_info)
    success = postgres.determine_setup()
    if not success:
        sys.exit(1)

    # default database name is <profile_name>_<login-name>
    # this ensures that for profiles named test_... the database will also
    # be named test_...
    import getpass
    osuser = getpass.getuser()
    dbname = db_name or profile_name + '_' + osuser

    # default database user name is aiida_qs_<login-name>
    # default password is random
    dbuser = db_username or 'aiida_qs_' + osuser
    from aiida.common.setup import generate_random_secret_key
    dbpass = db_password or generate_random_secret_key()

    # check if there is a profile that contains the db user already
    # and if yes, take the db user password from there
    # This is ok because a user can only see his own config files
    from aiida.common.setup import get_or_create_config
    confs = get_or_create_config()
    profs = confs.get('profiles', {})
    for profile in profs.itervalues():
        if profile.get('AIIDADB_USER', '') == dbuser and not db_password:
            dbpass = profile.get('AIIDADB_PASS')
            print 'using found password for {}'.format(dbuser)
            break

    try:
        create = True
        if not postgres.dbuser_exists(dbuser):
            postgres.create_dbuser(dbuser, dbpass)
        else:
            dbname, create = _check_db_name(dbname, postgres)
        if create:
            postgres.create_db(dbuser, dbname)
    except Exception as exception:
        click.echo('\n'.join([
            'Oops! Something went wrong while creating the database for you.',
            'You may continue with the quicksetup, however:',
            'For aiida to work correctly you will have to do that yourself as follows.',
            manual_setup_instructions(dbuser=dbuser, dbname=dbname), '',
            'Or setup your (OS-level) user to have permissions to create databases and rerun quicksetup.', ''
        ]))
        raise exception

    # create a profile, by default 'quicksetup' and prompt the user if
    # already exists
    confs = get_or_create_config()
    profile_name = profile_name or 'quicksetup'
    write_profile = False
    while not write_profile:
        if profile_name in confs.get('profiles', {}):
            if click.confirm('overwrite existing profile {}?'.format(profile_name)):
                write_profile = True
            else:
                profile_name = click.prompt('new profile name', type=str)
        else:
            write_profile = True

    dbhost = postgres.dbinfo.get('host', 'localhost')
    dbport = postgres.dbinfo.get('port', '5432')

    from os.path import isabs
    repo = repository or 'repository-{}/'.format(profile_name)
    if not isabs(repo):
        repo = os.path.join(aiida_dir, repo)

    setup_args = {
        'backend': backend,
        'email': email,
        'db_host': dbhost,
        'db_port': dbport,
        'db_name': dbname,
        'db_user': dbuser,
        'db_pass': dbpass,
        'repo': repo,
        'first_name': first_name,
        'last_name': last_name,
        'institution': institution,
        'force_overwrite': write_profile,
    }

    setup_profile(profile_name, only_config=only_config, set_default=set_default, non_interactive=True, **setup_args)
Esempio n. 4
0
class FixtureManager(object):
    """
    Manage the life cycle of a completely separated and temporary AiiDA environment

    * No previously created database of profile is required to run tests using this
      environment
    * Tests using this environment will never pollute the user's work environment

    Example::

        fixtures = FixtureManager()
        fixtures.create_aiida_db()  # set up only the database
        fixtures.create_profile()  # set up a profile (creates the db too if necessary)

        # ready for testing

        # run test 1

        fixtures.reset_db()
        # database ready for independent test 2

        # run test 2

        fixtures.destroy_all()
        # everything cleaned up

    Usage (unittest): See the :py:class:`PluginTestCase` and the :py:class:`TestRunner`.


    Usage (pytest)::

        import pytest

        @pytest.fixture(scope='session')
        def aiida_profile():
            with aiida.utils.fixtures.fixture_manager() as fixture_mgr:
                fixture_mgr.create_profile()
                yield fixture_manager

        @pytest.fixture(scope='function')
        def test_data(aiida_profile):
            # load my test data
            yield
            fixture_manager.reset_db()

        def test_my_stuff(test_data):
            # run a test


    """
    def __init__(self):
        self.db_params = {}
        self.fs_env = {'repo': 'test_repo', 'config': '.aiida'}
        self.profile_info = {
            'backend': 'django',
            'email': '*****@*****.**',
            'first_name': 'AiiDA',
            'last_name': 'Plugintest',
            'institution': 'aiidateam',
            'db_user': '******',
            'db_pass': '******',
            'db_name': 'aiida_db'
        }
        self.pg_cluster = None
        self.postgres = None
        self.__is_running_on_test_db = False
        self.__is_running_on_test_profile = False
        self._backup = {}
        self._backup['config_dir'] = aiida_cfg.AIIDA_CONFIG_FOLDER
        self._backup['profile'] = backend_settings.AIIDADB_PROFILE

    def create_db_cluster(self):
        if not self.pg_cluster:
            self.pg_cluster = PGTest(max_connections=256)
        self.db_params.update(self.pg_cluster.dsn)

    def create_aiida_db(self):
        """Create the necessary database on the temporary postgres instance"""
        if is_dbenv_loaded():
            raise FixtureError(
                'AiiDA dbenv can not be loaded while creating a test db environment'
            )
        if not self.db_params:
            self.create_db_cluster()
        self.postgres = Postgres(interactive=False, quiet=True)
        self.postgres.dbinfo = self.db_params
        self.postgres.determine_setup()
        self.db_params = self.postgres.dbinfo
        if not self.postgres.pg_execute:
            raise FixtureError(
                'Could not connect to the test postgres instance')
        self.postgres.create_dbuser(self.db_user, self.db_pass)
        self.postgres.create_db(self.db_user, self.db_name)
        self.__is_running_on_test_db = True

    def create_root_dir(self):
        self.root_dir = tempfile.mkdtemp()

    def create_profile(self):
        """
        Set AiiDA to use the test config dir and create a default profile there

        Warning: the AiiDA dbenv must not be loaded when this is called!
        """
        if is_dbenv_loaded():
            raise FixtureError(
                'AiiDA dbenv can not be loaded while creating a test profile')
        if not self.__is_running_on_test_db:
            self.create_aiida_db()
        from aiida.cmdline.verdilib import setup
        if not self.root_dir:
            self.create_root_dir()
        print(self.root_dir, self.config_dir)
        aiida_cfg.AIIDA_CONFIG_FOLDER = self.config_dir
        backend_settings.AIIDADB_PROFILE = None
        aiida_cfg.create_base_dirs()
        profile_name = 'test_profile'
        setup(profile=profile_name,
              only_config=False,
              non_interactive=True,
              **self.profile)
        aiida_cfg.set_default_profile('verdi', profile_name)
        aiida_cfg.set_default_profile('daemon', profile_name)
        self.__is_running_on_test_profile = True

    def reset_db(self):
        """Cleans all data from the database between tests"""
        if not self.__is_running_on_test_profile:
            raise FixtureError(
                'No test profile has been set up yet, can not reset the db')
        if self.profile_info['backend'] == BACKEND_DJANGO:
            self.__clean_db_django()
        elif self.profile_info['backend'] == BACKEND_SQLA:
            self.__clean_db_sqla()

    @property
    def profile(self):
        """Profile parameters"""
        profile = {
            'backend': self.backend,
            'email': self.email,
            'repo': self.repo,
            'db_host': self.db_host,
            'db_port': self.db_port,
            'db_user': self.db_user,
            'db_pass': self.db_pass,
            'db_name': self.db_name,
            'first_name': self.first_name,
            'last_name': self.last_name,
            'institution': self.institution
        }
        return profile

    @property
    def db_host(self):
        return self.db_params.get('host')

    @db_host.setter
    def db_host(self, hostname):
        self.db_params['host'] = hostname

    @property
    def first_name(self):
        return self.profile_info['first_name']

    @first_name.setter
    def first_name(self, name):
        self.profile_info['first_name'] = name

    @property
    def last_name(self):
        return self.profile_info['last_name']

    @last_name.setter
    def last_name(self, name):
        self.profile_info['last_name'] = name

    @property
    def institution(self):
        return self.profile_info['institution']

    @institution.setter
    def institution(self, institution):
        self.profile_info['institution'] = institution

    @property
    def db_port(self):
        return self.db_params.get('port', None)

    @db_port.setter
    def db_port(self, port):
        self.db_params['port'] = str(port)

    def repo_ok(self):
        return bool(self.repo and path.isdir(path.dirname(self.repo)))

    @property
    def repo(self):
        return self._return_dir('repo')

    @repo.setter
    def repo(self, repo_dir):
        self.fs_env['repo'] = repo_dir

    def _return_dir(self, key):
        """Return a path to a directory from the fs environment"""
        dir_path = self.fs_env[key]
        if not dir_path:
            raise FixtureError('no directory set for {}'.format(key))
        elif path.isabs(dir_path):
            return dir_path
        return path.join(self.root_dir, dir_path)

    @property
    def email(self):
        return self.profile_info['email']

    @email.setter
    def email(self, email):
        self.profile_info['email'] = email

    @property
    def backend(self):
        return self.profile_info['backend']

    @backend.setter
    def backend(self, backend):
        valid_backends = [BACKEND_DJANGO, BACKEND_SQLA]
        if backend not in valid_backends:
            raise ValueError('invalid backend {}, must be one of {}'.format(
                backend, valid_backends))
        self.profile_info['backend'] = backend

    @property
    def config_dir_ok(self):
        return bool(self.config_dir and path.isdir(self.config_dir))

    @property
    def config_dir(self):
        return self._return_dir('config')

    @config_dir.setter
    def config_dir(self, config_dir):
        self.fs_env['config'] = config_dir

    @property
    def db_user(self):
        return self.profile_info['db_user']

    @db_user.setter
    def db_user(self, user):
        self.profile_info['db_user'] = user

    @property
    def db_pass(self):
        return self.profile_info['db_pass']

    @db_pass.setter
    def db_pass(self, passwd):
        self.profile_info['db_pass'] = passwd

    @property
    def db_name(self):
        return self.profile_info['db_name']

    @db_name.setter
    def db_name(self, name):
        self.profile_info['db_name'] = name

    @property
    def root_dir(self):
        return self.fs_env.get('root', '')

    @root_dir.setter
    def root_dir(self, root_dir):
        self.fs_env['root'] = root_dir

    @property
    def root_dir_ok(self):
        return bool(self.root_dir and path.isdir(self.root_dir))

    def destroy_all(self):
        """Remove all traces of the test run"""
        if self.root_dir:
            shutil.rmtree(self.root_dir)
            self.root_dir = None
        if self.pg_cluster:
            self.pg_cluster.close()
            self.pg_cluster = None
        self.__is_running_on_test_db = False
        self.__is_running_on_test_profile = False
        if 'config_dir' in self._backup:
            aiida_cfg.AIIDA_CONFIG_FOLDER = self._backup['config_dir']
        if 'profile' in self._backup:
            backend_settings.AIIDADB_PROFILE = self._backup['profile']

    @staticmethod
    def __clean_db_django():
        from aiida.backends.djsite.db.testbase import DjangoTests
        DjangoTests().clean_db()

    def __clean_db_sqla(self):
        """Clean database for sqlalchemy backend"""
        from aiida.backends.sqlalchemy.tests.testbase import SqlAlchemyTests
        from aiida.backends.sqlalchemy import get_scoped_session
        from aiida.orm import User

        user = User.search_for_users(email=self.email)[0]
        new_user = User(email=user.email)
        new_user.first_name = user.first_name
        new_user.last_name = user.last_name
        new_user.institution = user.institution

        sqla_testcase = SqlAlchemyTests()
        sqla_testcase.test_session = get_scoped_session()
        sqla_testcase.clean_db()

        # that deleted our user, we need to recreate it
        new_user.force_save()

    def has_profile_open(self):
        return self.__is_running_on_test_profile
Esempio n. 5
0
class SetupTestCase(unittest.TestCase):
    """Test ``verdi setup``"""

    def setUp(self):
        self.runner = CliRunner()
        backend = os.environ.get('TEST_AIIDA_BACKEND', 'django')
        self.backend = 'django' if backend == 'django' else 'sqlalchemy'
        self.pg_test = PGTest()
        self.postgres = Postgres(port=self.pg_test.port, interactive=False, quiet=True)
        self.postgres.dbinfo = self.pg_test.dsn
        self.postgres.determine_setup()
        self.dbuser = '******'
        self.dbpass = '******'
        self.dbname = 'aiida_test_setup'
        self.postgres.create_dbuser(self.dbuser, self.dbpass)
        self.postgres.create_db(self.dbuser, self.dbname)
        self.repo = abspath('./aiida_radames')

    def tearDown(self):
        self.postgres.drop_db(self.dbname)
        self.postgres.drop_dbuser(self.dbuser)
        self.pg_test.close()

    def test_user_setup(self):
        backend_settings.AIIDADB_PROFILE = None
        result = self.runner.invoke(
            _setup_cmd, [
                'radames',
                '--non-interactive',
                '--backend={}'.format(self.backend),
                '[email protected]',
                '--first-name=Radames',
                '--last-name=Verdi',
                '--institution=Scala',
                '--repo={}'.format(self.repo),
                '--db_host=localhost',
                '--db_port={}'.format(self.pg_test.port),
                '--db_name={}'.format(self.dbname),
                '--db_user={}'.format(self.dbuser),
                '--db_pass={}'.format(self.dbpass),
                '--no-password'])
        self.assertFalse(result.exception, msg=get_debug_msg(result))

    def test_user_configure(self):
        backend_settings.AIIDADB_PROFILE = None
        self.runner.invoke(
            _setup_cmd, [
                'radames2',
                '--non-interactive',
                '--backend={}'.format(self.backend),
                '[email protected]',
                '--first-name=Radames',
                '--last-name=Verdi',
                '--institution=Scala',
                '--repo={}'.format(self.repo),
                '--db_host=localhost',
                '--db_port={}'.format(self.pg_test.port),
                '--db_name={}'.format(self.dbname),
                '--db_user={}'.format(self.dbuser),
                '--db_pass={}'.format(self.dbpass),
                '--no-password'])

        backend_settings.AIIDADB_PROFILE = None
        result = self.runner.invoke(
            _setup_cmd,
            ['radames2', '--only-config'],
            input='yes\[email protected]\npostgresql_psycopg2\n\n\n\n\n\n{repo}\nRadames2\nVerdi2\nScala2\nyes\nno\n'.format(
                repo=self.repo
            ),
            catch_exceptions=False
        )
        self.assertFalse(result.exception, msg=get_debug_msg(result))
Esempio n. 6
0
class SetupTestCase(unittest.TestCase):
    """Test `verdi setup`."""
    def setUp(self):
        self.runner = CliRunner()
        backend = os.environ.get('TEST_AIIDA_BACKEND', 'django')
        self.backend = 'django' if backend == 'django' else 'sqlalchemy'
        self.pg_test = PGTest()
        self.postgres = Postgres(port=self.pg_test.port,
                                 interactive=False,
                                 quiet=True)
        self.postgres.dbinfo = self.pg_test.dsn
        self.postgres.determine_setup()
        self.dbuser = '******'
        self.dbpass = '******'
        self.dbname = 'aiida_test_setup_{}'.format(self.backend)
        self.postgres.create_dbuser(self.dbuser, self.dbpass)
        self.postgres.create_db(self.dbuser, self.dbname)
        self.repository = abspath('./aiida_radames_{}'.format(self.backend))

    def tearDown(self):
        self.postgres.drop_db(self.dbname)
        self.postgres.drop_dbuser(self.dbuser)
        self.pg_test.close()

    def test_user_setup(self):
        """
        Test `verdi setup` non-interactively
        """
        backend_settings.AIIDADB_PROFILE = None
        result = self.runner.invoke(setup, [
            '--non-interactive', '--backend={}'.format(
                self.backend), '[email protected]',
            '--first-name=Radames', '--last-name=Verdi', '--institution=Scala',
            '--repository={}'.format(self.repository), '--db-host=localhost',
            '--db-port={}'.format(self.pg_test.port), '--db-name={}'.format(
                self.dbname), '--db-username={}'.format(
                    self.dbuser), '--db-password={}'.format(
                        self.dbpass), 'radames_{}'.format(self.backend)
        ])
        self.assertFalse(result.exception, msg=get_debug_msg(result))

    def test_user_configure(self):
        """
        Test `verdi setup` configure user
        """
        backend_settings.AIIDADB_PROFILE = None
        self.runner.invoke(setup, [
            '--non-interactive', '--backend={}'.format(
                self.backend), '[email protected]',
            '--first-name=Radames', '--last-name=Verdi', '--institution=Scala',
            '--repository={}'.format(self.repository), '--db-host=localhost',
            '--db-port={}'.format(self.pg_test.port), '--db-name={}'.format(
                self.dbname), '--db-username={}'.format(
                    self.dbuser), '--db-password={}'.format(
                        self.dbpass), 'radames2_{}'.format(self.backend)
        ])

        tpl = '{email}\n{first_name}\n{last_name}\n{institution}\nyes\n{email}\n{engine}\n\n\n\n\n\n{repo}\nno\n\n'
        backend_settings.AIIDADB_PROFILE = None
        result = self.runner.invoke(
            setup, ['radames2_{}'.format(self.backend), '--only-config'],
            input=tpl.format(email='*****@*****.**',
                             first_name='Radames2',
                             last_name='Verdi2',
                             institution='Scala2',
                             engine='postgresql_psycopg2',
                             repo=self.repository),
            catch_exceptions=False)
        self.assertFalse(result.exception, msg=get_debug_msg(result))