def computer_setup(ctx, non_interactive, **kwargs):
"""Create a new computer."""
from aiida.orm.utils.builders.computer import ComputerBuilder
if kwargs['label'] in get_computer_names():
echo.echo_critical(
'A computer called {c} already exists. '
'Use "verdi computer duplicate {c}" to set up a new '
'computer starting from the settings of {c}.'.format(
c=kwargs['label']))
kwargs['transport'] = kwargs['transport'].name
kwargs['scheduler'] = kwargs['scheduler'].name
computer_builder = ComputerBuilder(**kwargs)
try:
computer = computer_builder.new()
except (ComputerBuilder.ComputerValidationError, ValidationError) as e:
echo.echo_critical(f'{type(e).__name__}: {e}')
try:
computer.store()
except ValidationError as err:
echo.echo_critical(f'unable to store the computer: {err}. Exiting...')
else:
echo.echo_success(f'Computer<{computer.pk}> {computer.label} created')
echo.echo_info(
'Note: before the computer can be used, it has to be configured with the command:'
)
echo.echo_info(
f' verdi computer configure {computer.transport_type} {computer.label}'
)
def on_setup_computer(self, _=None, on_success=None):
"""Create a new computer."""
with self.setup_compupter_out:
clear_output()
if self.label.value == "": # check hostname
print("Please specify the computer name (for AiiDA)")
return
try:
computer = orm.Computer.objects.get(label=self.label.value)
print(f"A computer called {self.label.value} already exists.")
if on_success:
on_success()
return
except common.NotExistent:
pass
items_to_configure = [
"label",
"hostname",
"description",
"work_dir",
"mpirun_command",
"mpiprocs_per_machine",
"transport",
"scheduler",
"prepend_text",
"append_text",
"shebang",
]
kwargs = {
key: getattr(self, key).value
for key in items_to_configure
}
computer_builder = ComputerBuilder(**kwargs)
try:
computer = computer_builder.new()
except (
ComputerBuilder.ComputerValidationError,
common.exceptions.ValidationError,
) as err:
print(f"{type(err).__name__}: {err}")
return
try:
computer.store()
except common.exceptions.ValidationError as err:
print(f"Unable to store the computer: {err}.")
return
if self._configure_computer():
if on_success:
on_success()
print(f"Computer<{computer.pk}> {computer.label} created")
def setUp(self):
"""Prepare current user and computer builder with common properties."""
from aiida.orm.utils.builders.computer import ComputerBuilder
self.comp_builder = ComputerBuilder(label='test', description='computer', hostname='localhost')
self.comp_builder.scheduler = 'direct'
self.comp_builder.work_dir = '/tmp/aiida'
self.comp_builder.prepend_text = ''
self.comp_builder.append_text = ''
self.comp_builder.mpiprocs_per_machine = 8
self.comp_builder.mpirun_command = 'mpirun'
self.comp_builder.shebang = '#!xonsh'
self.user = orm.User.objects.get_default()
class TestComputerConfigure(AiidaTestCase):
"""Tests for the configuring of instance of the `Computer` ORM class."""
def setUp(self):
"""Prepare current user and computer builder with common properties."""
from aiida.orm.utils.builders.computer import ComputerBuilder
self.comp_builder = ComputerBuilder(label='test',
description='computer',
hostname='localhost')
self.comp_builder.scheduler = 'direct'
self.comp_builder.work_dir = '/tmp/aiida'
self.comp_builder.prepend_text = ''
self.comp_builder.append_text = ''
self.comp_builder.mpiprocs_per_machine = 8
self.comp_builder.mpirun_command = 'mpirun'
self.comp_builder.shebang = '#!xonsh'
self.user = orm.User.objects.get_default()
def test_configure_local(self):
"""Configure a computer for local transport and check it is configured."""
self.comp_builder.label = 'test_configure_local'
self.comp_builder.transport = 'local'
comp = self.comp_builder.new()
comp.store()
comp.configure()
self.assertTrue(comp.is_user_configured(self.user))
def test_configure_ssh(self):
"""Configure a computer for ssh transport and check it is configured."""
self.comp_builder.label = 'test_configure_ssh'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
comp.configure(username='******', port='22')
self.assertTrue(comp.is_user_configured(self.user))
def test_configure_ssh_invalid(self):
"""Try to configure computer with invalid auth params and check it fails."""
self.comp_builder.label = 'test_configure_ssh_invalid'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
with self.assertRaises(ValueError):
comp.configure(username='******', invalid_auth_param='TEST')
def computer_setup(ctx, non_interactive, **kwargs):
"""Create a new computer."""
from aiida.orm.utils.builders.computer import ComputerBuilder
if kwargs['label'] in get_computer_names():
echo.echo_critical(
'A computer called {c} already exists. '
'Use "verdi computer duplicate {c}" to set up a new '
'computer starting from the settings of {c}.'.format(
c=kwargs['label']))
if not non_interactive:
try:
pre, post = ensure_scripts(kwargs.pop('prepend_text', ''),
kwargs.pop('append_text', ''), kwargs)
except InputValidationError as exception:
raise click.BadParameter(
'invalid prepend and or append text: {}'.format(exception))
kwargs['prepend_text'] = pre
kwargs['append_text'] = post
kwargs['transport'] = kwargs['transport'].name
kwargs['scheduler'] = kwargs['scheduler'].name
computer_builder = ComputerBuilder(**kwargs)
try:
computer = computer_builder.new()
except (ComputerBuilder.ComputerValidationError, ValidationError) as e:
echo.echo_critical('{}: {}'.format(type(e).__name__, e))
try:
computer.store()
except ValidationError as err:
echo.echo_critical(
'unable to store the computer: {}. Exiting...'.format(err))
else:
echo.echo_success('Computer<{}> {} created'.format(
computer.pk, computer.name))
echo.echo_info(
'Note: before the computer can be used, it has to be configured with the command:'
)
echo.echo_info(' verdi computer configure {} {}'.format(
computer.get_transport_type(), computer.name))
def get_cx1_computer(work_dir=None, key_filename=None):
"""get or create a Computer node, to interface with the ICL Cx1 HPC
Parameters
----------
work_dir :str
absolute path to the work directory
key_filename : str
ssh key filename, e.g. /Users/user_name/.ssh/id_rsa
Returns
-------
aiida.orm.Computer
"""
try:
computer_cx1 = Computer.objects.get(name=_COMPUTER_NAME)
except NotExistent:
if work_dir is None:
raise ValueError('work_dir not specified')
computer_builder = ComputerBuilder(
label=_COMPUTER_NAME,
description='Imperial HPC cx1 computer',
transport='ssh',
scheduler='pbspro_cx1',
hostname='login.cx1.hpc.ic.ac.uk',
prepend_text='',
append_text='',
work_dir=work_dir,
shebang='#!/bin/bash',
mpiprocs_per_machine=16,
mpirun_command='mpiexec')
computer_cx1 = computer_builder.new()
computer_cx1.store()
computer_cx1.configure(look_for_keys=True,
key_filename=key_filename,
timeout=60,
allow_agent=True,
compress=True,
load_system_host_keys=True,
safe_interval=5.0)
return computer_cx1
class TestVerdiComputerConfigure(AiidaTestCase):
"""Test the ``verdi computer configure`` command."""
def setUp(self):
"""Prepare computer builder with common properties."""
from aiida.orm.utils.builders.computer import ComputerBuilder
self.cli_runner = CliRunner()
self.user = orm.User.objects.get_default()
self.comp_builder = ComputerBuilder(label='test_comp_setup')
self.comp_builder.hostname = 'localhost'
self.comp_builder.description = 'Test Computer'
self.comp_builder.scheduler = 'direct'
self.comp_builder.work_dir = '/tmp/aiida'
self.comp_builder.prepend_text = ''
self.comp_builder.append_text = ''
self.comp_builder.mpiprocs_per_machine = 8
self.comp_builder.mpirun_command = 'mpirun'
self.comp_builder.shebang = '#!xonsh'
def test_top_help(self):
"""Test help option of verdi computer configure."""
result = self.cli_runner.invoke(computer_configure, ['--help'], catch_exceptions=False)
self.assertIn('ssh', result.output)
self.assertIn('local', result.output)
def test_reachable(self): # pylint: disable=no-self-use
"""Test reachability of top level and sub commands."""
import subprocess as sp
sp.check_output(['verdi', 'computer', 'configure', '--help'])
sp.check_output(['verdi', 'computer', 'configure', 'local', '--help'])
sp.check_output(['verdi', 'computer', 'configure', 'ssh', '--help'])
sp.check_output(['verdi', 'computer', 'configure', 'show', '--help'])
def test_local_ni_empty(self):
"""
Test verdi computer configure local <comp>
Test twice, with comp setup for local or ssh.
* with computer setup for local: should succeed
* with computer setup for ssh: should fail
"""
self.comp_builder.label = 'test_local_ni_empty'
self.comp_builder.transport = 'local'
comp = self.comp_builder.new()
comp.store()
options = ['local', comp.label, '--non-interactive', '--safe-interval', '0']
result = self.cli_runner.invoke(computer_configure, options, catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
self.comp_builder.label = 'test_local_ni_empty_mismatch'
self.comp_builder.transport = 'ssh'
comp_mismatch = self.comp_builder.new()
comp_mismatch.store()
options = ['local', comp_mismatch.label, '--non-interactive']
result = self.cli_runner.invoke(computer_configure, options, catch_exceptions=False)
self.assertIsNotNone(result.exception)
self.assertIn('ssh', result.output)
self.assertIn('local', result.output)
def test_local_interactive(self):
"""Test computer configuration for local transports."""
self.comp_builder.label = 'test_local_interactive'
self.comp_builder.transport = 'local'
comp = self.comp_builder.new()
comp.store()
result = self.cli_runner.invoke(computer_configure, ['local', comp.label], input='\n', catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
def test_ssh_interactive(self):
"""
Check that the interactive prompt is accepting the correct values.
Actually, even passing a shorter set of options should work:
``verdi computer configure ssh`` is able to provide sensible default
parameters reading from the ssh config file.
We are here therefore only checking some of them.
"""
self.comp_builder.label = 'test_ssh_interactive'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
remote_username = '******'
port = 345
look_for_keys = False
key_filename = ''
# I just pass the first four arguments:
# the username, the port, look_for_keys, and the key_filename
# This testing also checks that an empty key_filename is ok
command_input = ('{remote_username}\n{port}\n{look_for_keys}\n{key_filename}\n').format(
remote_username=remote_username,
port=port,
look_for_keys='yes' if look_for_keys else 'no',
key_filename=key_filename
)
result = self.cli_runner.invoke(
computer_configure, ['ssh', comp.label], input=command_input, catch_exceptions=False
)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
new_auth_params = comp.get_authinfo(self.user).get_auth_params()
self.assertEqual(new_auth_params['username'], remote_username)
self.assertEqual(new_auth_params['port'], port)
self.assertEqual(new_auth_params['look_for_keys'], look_for_keys)
self.assertEqual(new_auth_params['key_filename'], key_filename)
def test_local_from_config(self):
"""Test configuring a computer from a config file"""
label = 'test_local_from_config'
self.comp_builder.label = label
self.comp_builder.transport = 'local'
computer = self.comp_builder.new()
computer.store()
interval = 20
with tempfile.NamedTemporaryFile('w') as handle:
handle.write("""---
safe_interval: {interval}
""".format(interval=interval))
handle.flush()
options = ['local', computer.label, '--config', os.path.realpath(handle.name)]
result = self.cli_runner.invoke(computer_configure, options)
self.assertClickResultNoException(result)
self.assertEqual(computer.get_configuration()['safe_interval'], interval)
def test_ssh_ni_empty(self):
"""
Test verdi computer configure ssh <comp>
Test twice, with comp setup for ssh or local.
* with computer setup for ssh: should succeed
* with computer setup for local: should fail
"""
self.comp_builder.label = 'test_ssh_ni_empty'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
options = ['ssh', comp.label, '--non-interactive', '--safe-interval', '1']
result = self.cli_runner.invoke(computer_configure, options, catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
self.comp_builder.label = 'test_ssh_ni_empty_mismatch'
self.comp_builder.transport = 'local'
comp_mismatch = self.comp_builder.new()
comp_mismatch.store()
options = ['ssh', comp_mismatch.label, '--non-interactive']
result = self.cli_runner.invoke(computer_configure, options, catch_exceptions=False)
self.assertIsNotNone(result.exception)
self.assertIn('local', result.output)
self.assertIn('ssh', result.output)
def test_ssh_ni_username(self):
"""Test verdi computer configure ssh <comp> --username=<username>."""
self.comp_builder.label = 'test_ssh_ni_username'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
username = '******'
options = ['ssh', comp.label, '--non-interactive', '--username={}'.format(username), '--safe-interval', '1']
result = self.cli_runner.invoke(computer_configure, options, catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
self.assertEqual(
orm.AuthInfo.objects.get(dbcomputer_id=comp.id, aiidauser_id=self.user.id).get_auth_params()['username'],
username
)
def test_show(self):
"""Test verdi computer configure show <comp>."""
self.comp_builder.label = 'test_show'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
result = self.cli_runner.invoke(computer_configure, ['show', comp.label], catch_exceptions=False)
result = self.cli_runner.invoke(computer_configure, ['show', comp.label, '--defaults'], catch_exceptions=False)
self.assertIn('* username', result.output)
result = self.cli_runner.invoke(
computer_configure, ['show', comp.label, '--defaults', '--as-option-string'], catch_exceptions=False
)
self.assertIn('--username='******'ssh', comp.label, '--non-interactive']
config_cmd.extend(result.output.replace("'", '').split(' '))
result_config = self.cli_runner.invoke(computer_configure, config_cmd, catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result_config.output)
result_cur = self.cli_runner.invoke(
computer_configure, ['show', comp.label, '--as-option-string'], catch_exceptions=False
)
self.assertIn('--username=', result.output)
self.assertEqual(result_cur.output, result.output)
class TestVerdiComputerConfigure(AiidaTestCase):
def setUp(self):
"""Prepare computer builder with common properties."""
from aiida.orm.utils.builders.computer import ComputerBuilder
self.cli_runner = CliRunner()
self.user = orm.User.objects.get_default()
self.comp_builder = ComputerBuilder(label='test_comp_setup')
self.comp_builder.hostname = 'localhost'
self.comp_builder.description = 'Test Computer'
self.comp_builder.scheduler = 'direct'
self.comp_builder.work_dir = '/tmp/aiida'
self.comp_builder.prepend_text = ''
self.comp_builder.append_text = ''
self.comp_builder.mpiprocs_per_machine = 8
self.comp_builder.mpirun_command = 'mpirun'
self.comp_builder.shebang = '#!xonsh'
def test_top_help(self):
"""Test help option of verdi computer configure."""
result = self.cli_runner.invoke(computer_configure, ['--help'],
catch_exceptions=False)
self.assertIn('ssh', result.output)
self.assertIn('local', result.output)
def test_reachable(self):
"""Test reachability of top level and sub commands."""
import subprocess as sp
output = sp.check_output(['verdi', 'computer', 'configure', '--help'])
sp.check_output(['verdi', 'computer', 'configure', 'local', '--help'])
sp.check_output(['verdi', 'computer', 'configure', 'ssh', '--help'])
sp.check_output(['verdi', 'computer', 'configure', 'show', '--help'])
def test_local_ni_empty(self):
"""
Test verdi computer configure local <comp>
Test twice, with comp setup for local or ssh.
* with computer setup for local: should succeed
* with computer setup for ssh: should fail
"""
self.comp_builder.label = 'test_local_ni_empty'
self.comp_builder.transport = 'local'
comp = self.comp_builder.new()
comp.store()
options = [
'local', comp.label, '--non-interactive', '--safe-interval', '0'
]
result = self.cli_runner.invoke(computer_configure,
options,
catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
self.comp_builder.label = 'test_local_ni_empty_mismatch'
self.comp_builder.transport = 'ssh'
comp_mismatch = self.comp_builder.new()
comp_mismatch.store()
options = ['local', comp_mismatch.label, '--non-interactive']
result = self.cli_runner.invoke(computer_configure,
options,
catch_exceptions=False)
self.assertIsNotNone(result.exception)
self.assertIn('ssh', result.output)
self.assertIn('local', result.output)
def test_local_interactive(self):
self.comp_builder.label = 'test_local_interactive'
self.comp_builder.transport = 'local'
comp = self.comp_builder.new()
comp.store()
result = self.cli_runner.invoke(computer_configure,
['local', comp.label],
input='\n',
catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
def test_local_from_config(self):
"""Test configuring a computer from a config file"""
label = 'test_local_from_config'
self.comp_builder.label = label
self.comp_builder.transport = 'local'
computer = self.comp_builder.new()
computer.store()
interval = 20
with tempfile.NamedTemporaryFile('w') as handle:
handle.write("""---
safe_interval: {interval}
""".format(interval=interval))
handle.flush()
options = [
'local', computer.label, '--config',
os.path.realpath(handle.name)
]
result = self.cli_runner.invoke(computer_configure, options)
self.assertClickResultNoException(result)
self.assertEqual(computer.get_configuration()['safe_interval'],
interval)
def test_ssh_ni_empty(self):
"""
Test verdi computer configure ssh <comp>
Test twice, with comp setup for ssh or local.
* with computer setup for ssh: should succeed
* with computer setup for local: should fail
"""
self.comp_builder.label = 'test_ssh_ni_empty'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
options = [
'ssh', comp.label, '--non-interactive', '--safe-interval', '1'
]
result = self.cli_runner.invoke(computer_configure,
options,
catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
self.comp_builder.label = 'test_ssh_ni_empty_mismatch'
self.comp_builder.transport = 'local'
comp_mismatch = self.comp_builder.new()
comp_mismatch.store()
options = ['ssh', comp_mismatch.label, '--non-interactive']
result = self.cli_runner.invoke(computer_configure,
options,
catch_exceptions=False)
self.assertIsNotNone(result.exception)
self.assertIn('local', result.output)
self.assertIn('ssh', result.output)
def test_ssh_ni_username(self):
"""Test verdi computer configure ssh <comp> --username=<username>."""
self.comp_builder.label = 'test_ssh_ni_username'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
username = '******'
options = [
'ssh', comp.label, '--non-interactive',
'--username={}'.format(username), '--safe-interval', '1'
]
result = self.cli_runner.invoke(computer_configure,
options,
catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user), msg=result.output)
self.assertEqual(
orm.AuthInfo.objects.get(
dbcomputer_id=comp.id,
aiidauser_id=self.user.id).get_auth_params()['username'],
username)
def test_show(self):
"""Test verdi computer configure show <comp>."""
self.comp_builder.label = 'test_show'
self.comp_builder.transport = 'ssh'
comp = self.comp_builder.new()
comp.store()
result = self.cli_runner.invoke(computer_configure,
['show', comp.label],
catch_exceptions=False)
result = self.cli_runner.invoke(computer_configure,
['show', comp.label, '--defaults'],
catch_exceptions=False)
self.assertIn('* username', result.output)
result = self.cli_runner.invoke(
computer_configure,
['show', comp.label, '--defaults', '--as-option-string'],
catch_exceptions=False)
self.assertIn('--username='******'ssh', comp.label, '--non-interactive']
config_cmd.extend(result.output.replace("'", '').split(' '))
result_config = self.cli_runner.invoke(computer_configure,
config_cmd,
catch_exceptions=False)
self.assertTrue(comp.is_user_configured(self.user),
msg=result_config.output)
result_cur = self.cli_runner.invoke(
computer_configure, ['show', comp.label, '--as-option-string'],
catch_exceptions=False)
self.assertIn('--username=', result.output)
self.assertEqual(result_cur.output, result.output)
def set_computer_builder(ctx, param, value):
"""Set the computer spec for defaults of following options."""
from aiida.orm.utils.builders.computer import ComputerBuilder
ctx.computer_builder = ComputerBuilder.from_computer(value)
return value
def duplicate_fe(computer: Computer, include_codes, input_plugin, suffix,
dry_run, job_should_keep_env):
"""
Create copies of the existing computer using FwScheduler, add existing
Code if requested.
"""
from aiida import orm
from aiida.orm.utils.builders.computer import ComputerBuilder
from fireworks.fw_config import LAUNCHPAD_LOC
if LAUNCHPAD_LOC is None:
echo.echo_critical(
'Cannot find the default Fireworks launchpad. '
'Please make sure you have configured Fireworks correctly, see '
'https://materialsproject.github.io/fireworks/config_tutorial.html'
)
builder = ComputerBuilder.from_computer(computer)
if 'slurm' in computer.scheduler_type or job_should_keep_env:
builder.scheduler = "fireworks_scheduler.keepenv"
else:
builder.scheduler = "fireworks_scheduler.default"
echo.echo_info(f"Scheduler for the new computer: {builder.scheduler}")
builder.label += "-" + suffix
builder.description += "(Using Fireworks as the scheduler.)"
comp = builder.new()
echo.echo_info(f"Adding new computer {comp}")
if not dry_run:
comp.store()
echo.echo_info(f"Computer {comp} has been saved to the database")
if include_codes:
qb_code_filters = dict()
if input_plugin:
qb_code_filters['attributes.input_plugin'] = input_plugin.name
user = orm.User.objects.get_default()
qbd = orm.QueryBuilder()
qbd.append(orm.Computer, tag='computer', filters={'id': computer.pk})
qbd.append(orm.Code,
with_computer='computer',
tag='code',
filters=qb_code_filters,
project=['*'])
qbd.append(orm.User,
tag='user',
with_node='code',
filters={'email': user.email})
new_codes = []
for (code, ) in qbd.iterall():
new_code = deepcopy(code)
new_code.set_remote_computer_exec(
(comp, code.get_remote_exec_path()))
new_codes.append(new_code)
echo.echo_info(f"Adding new code {new_code}")
if not dry_run:
for code in new_codes:
code.store()
echo.echo_info(f"Code {code} has been saved to the database")
if dry_run:
echo.echo_info("This is a dry-run nothing has been saved.")
