def insert_data(self):
"""
Insert default data into the DB.
"""
from django.core.exceptions import ObjectDoesNotExist
from aiida.backends.djsite.db.models import DbUser
from aiida.orm.computer import Computer
from aiida.common.utils import get_configured_user_email
# We create the user only once:
# Otherwise, get_automatic_user() will fail when the
# user is recreated because it caches the user!
# In any case, store it in self.user though
try:
self.user = DbUser.objects.get(email=get_configured_user_email())
except ObjectDoesNotExist:
self.user = DbUser.objects.create_user(get_configured_user_email(),
'fakepwd')
# Reqired by the calling class
self.user_email = self.user.email
# Also self.computer is required by the calling class
self.computer = Computer(name='localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
self.computer.store()
def setUpClass(cls):
from aiida.orm.user import User
from aiida.orm.computer import Computer
from aiida.common.utils import get_configured_user_email
# I create the user only once:
# We create the user only once:
# Otherwise, get_automatic_user() will fail when the
# user is recreated because it caches the user!
# In any case, store it in cls.user though
# Other possibility: flush the user cache on delete
users = User.search_for_users(email=get_configured_user_email())
assert (len(users) <= 1)
if users:
cls.user = users[0]
else:
cls.user = User(get_configured_user_email())
cls.user.password = '******'
cls.user.save()
try:
cls.computer = Computer.get('test_comp')
except NotExistent:
cls.computer = Computer(name='test_comp',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
cls.computer.store()
def test_session_update_and_expiration_2(self):
"""
expire_on_commit=True & committing computer and code objects with
their built-in store function.
"""
from aiida.backends.sqlalchemy.models.user import DbUser
from aiida.orm.computer import Computer
from aiida.orm.code import Code
from aiida.orm.user import User
session = aiida.backends.sqlalchemy.get_scoped_session()
self.set_connection(expire_on_commit=True)
user = User(email=get_configured_user_email())
session.add(user._dbuser)
session.commit()
defaults = dict(name='localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
computer = Computer(**defaults)
computer.store()
code = Code()
code.set_remote_computer_exec((computer, '/x.x'))
code.store()
self.drop_connection()
def test_computer_delete(self):
"""
Test if 'verdi computer delete' command works
"""
from aiida.orm.computer import Computer as AiidaOrmComputer
from aiida.common.exceptions import NotExistent
# Setup a computer to delete during the test
comp = AiidaOrmComputer(name='computer_for_test_delete',
hostname='localhost',
transport_type='local',
scheduler_type='direct',
workdir='/tmp/aiida')
comp.store()
# See if the command complains about not getting an invalid computer
options = ['non_existent_computer_name']
result = self.runner.invoke(computer_delete, options)
# Exception should be raised
self.assertIsNotNone(result.exception)
# Delete a computer name successully.
options = ['computer_for_test_delete']
result = self.runner.invoke(computer_delete, options)
# Exception should be not be raised
self.assertIsNone(result.exception)
# Check that the computer really was deleted
with self.assertRaises(NotExistent):
AiidaOrmComputer.get('computer_for_test_delete')
def computer_setup(self, *args):
"""
Setup a new or existing computer
"""
import readline
if len(args) != 0:
print >> sys.stderr, ("after 'computer setup' there cannot be any "
"argument.")
sys.exit(1)
if not is_dbenv_loaded():
load_dbenv()
from aiida.common.exceptions import NotExistent, ValidationError
from aiida.orm.computer import Computer as AiidaOrmComputer
print "At any prompt, type ? to get some help."
print "---------------------------------------"
# get the new computer name
readline.set_startup_hook(lambda: readline.insert_text(previous_value))
input_txt = raw_input("=> Computer name: ")
if input_txt.strip() == '?':
print "HELP:", "The computer name"
computer_name = input_txt.strip()
try:
computer = self.get_computer(name=computer_name)
print "A computer called {} already exists.".format(computer_name)
print "Use 'verdi computer update' to update it, and be careful if"
print "you really want to modify a database entry!"
print "Now exiting..."
sys.exit(1)
except NotExistent:
computer = AiidaOrmComputer(name=computer_name)
print "Creating new computer with name '{}'".format(computer_name)
prompt_for_computer_configuration(computer)
try:
computer.store()
except ValidationError as e:
print "Unable to store the computer: {}. Exiting...".format(
e.message)
sys.exit(1)
print "Computer '{}' successfully stored in DB.".format(computer_name)
print "pk: {}, uuid: {}".format(computer.pk, computer.uuid)
print "Note: before using it with AiiDA, configure it using the command"
print " verdi computer configure {}".format(computer_name)
print "(Note: machine_dependent transport parameters cannot be set via "
print "the command-line interface at the moment)"
def setUpClass(cls):
from django.core.exceptions import ObjectDoesNotExist
from aiida.backends.djsite.db.models import DbUser
from aiida.orm.computer import Computer
from aiida.common.utils import get_configured_user_email
# I create the user only once:
# We create the user only once:
# Otherwise, get_automatic_user() will fail when the
# user is recreated because it caches the user!
# In any case, store it in cls.user though
# Other possibility: flush the user cache on delete
try:
cls.user = DbUser.objects.get(email=get_configured_user_email())
except ObjectDoesNotExist:
cls.user = DbUser.objects.create_user(get_configured_user_email(),
'fakepwd')
cls.computer = Computer(name='localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
cls.computer.store()
def get_computer(self, name):
"""
Get a Computer object with given name, or raise NotExistent
"""
from aiida.orm.computer import Computer as AiidaOrmComputer
return AiidaOrmComputer.get(name)
def test_session_update_and_expiration_3(self):
"""
expire_on_commit=False & adding manually and committing
computer and code objects.
"""
from aiida.orm.computer import Computer
from aiida.orm.code import Code
from aiida.orm.user import User
self.set_connection(expire_on_commit=False)
session = aiida.backends.sqlalchemy.get_scoped_session()
user = User(email=get_configured_user_email())
session.add(user._dbuser)
session.commit()
defaults = dict(name='localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
computer = Computer(**defaults)
session.add(computer._dbcomputer)
session.commit()
code = Code()
code.set_remote_computer_exec((computer, '/x.x'))
session.add(code.dbnode)
session.commit()
self.drop_connection()
def test_deletion(self):
from aiida.orm.computer import Computer
from aiida.orm import delete_computer, JobCalculation
from aiida.common.exceptions import InvalidOperation
newcomputer = Computer(name="testdeletioncomputer", hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida').store()
# # This should be possible, because nothing is using this computer
delete_computer(newcomputer)
calc_params = {
'computer': self.computer,
'resources': {'num_machines': 1,
'num_mpiprocs_per_machine': 1}
}
_ = JobCalculation(**calc_params).store()
# This should fail, because there is at least a calculation
# using this computer (the one created just above)
with self.assertRaises(InvalidOperation):
delete_computer(self.computer)
def insert_data(self):
"""
Insert default data into the DB.
"""
email = get_configured_user_email()
has_user = DbUser.query.filter(DbUser.email == email).first()
if not has_user:
self.user = DbUser(get_configured_user_email(), "foo", "bar",
"tests")
self.test_session.add(self.user)
self.test_session.commit()
else:
self.user = has_user
# Required by the calling class
self.user_email = self.user.email
# Also self.computer is required by the calling class
has_computer = DbComputer.query.filter(
DbComputer.hostname == 'localhost').first()
if not has_computer:
self.computer = SqlAlchemyTests._create_computer()
self.computer.store()
else:
self.computer = Computer(dbcomputer=has_computer)
def _create_computer(**kwargs):
defaults = dict(name='localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
defaults.update(kwargs)
return Computer(**defaults)
def test_computer_rename(self):
"""
Test if 'verdi computer rename' command works
"""
from aiida.orm.computer import Computer as AiidaOrmComputer
from aiida.common.exceptions import NotExistent
# See if the command complains about not getting an invalid computer
options = ['not_existent_computer_name']
result = self.runner.invoke(computer_rename, options)
# Exception should be raised
self.assertIsNotNone(result.exception)
# See if the command complains about not getting both names
options = ['comp_cli_test_computer']
result = self.runner.invoke(computer_rename, options)
# Exception should be raised
self.assertIsNotNone(result.exception)
# The new name must be different to the old one
options = ['comp_cli_test_computer', 'comp_cli_test_computer']
result = self.runner.invoke(computer_rename, options)
# Exception should be raised
self.assertIsNotNone(result.exception)
# Change a computer name successully.
options = ['comp_cli_test_computer', 'renamed_test_computer']
result = self.runner.invoke(computer_rename, options)
# Exception should be not be raised
self.assertIsNone(result.exception)
# Check that the name really was changed
# The old name should not be available
with self.assertRaises(NotExistent):
AiidaOrmComputer.get('comp_cli_test_computer')
# The new name should be avilable
try:
AiidaOrmComputer.get('renamed_test_computer')
except NotExistent:
assertTrue(False)
# Now change the name back
options = ['renamed_test_computer', 'comp_cli_test_computer']
result = self.runner.invoke(computer_rename, options)
# Exception should be not be raised
self.assertIsNone(result.exception)
# Check that the name really was changed
# The old name should not be available
with self.assertRaises(NotExistent):
AiidaOrmComputer.get('renamed_test_computer')
# The new name should be avilable
try:
AiidaOrmComputer.get('comp_cli_test_computer')
except NotExistent:
assertTrue(False)
def get_computer(self):
"""
Get the computer associated to the node.
:return: the Computer object or None.
"""
from aiida.orm.computer import Computer
if self.dbnode.dbcomputer is None:
return None
else:
return Computer(dbcomputer=self.dbnode.dbcomputer)
def test_remote(self):
import tempfile
from aiida.orm.code import Code
from aiida.orm.computer import Computer
from aiida.common.exceptions import ValidationError
with self.assertRaises(ValueError):
# remote_computer_exec has length 2 but is not a list or tuple
_ = Code(remote_computer_exec='ab')
# invalid code path
with self.assertRaises(ValueError):
_ = Code(remote_computer_exec=(self.computer, ''))
# Relative path is invalid for remote code
with self.assertRaises(ValueError):
_ = Code(remote_computer_exec=(self.computer, 'subdir/run.exe'))
# first argument should be a computer, not a string
with self.assertRaises(TypeError):
_ = Code(remote_computer_exec=('localhost', '/bin/ls'))
code = Code(remote_computer_exec=(self.computer, '/bin/ls'))
with tempfile.NamedTemporaryFile() as f:
f.write("#/bin/bash\n\necho test run\n")
f.flush()
code.add_path(f.name, 'test.sh')
with self.assertRaises(ValidationError):
# There are files inside
code.store()
# If there are no files, I can store
code.remove_path('test.sh')
code.store()
self.assertEquals(code.get_remote_computer().pk, self.computer.pk)
self.assertEquals(code.get_remote_exec_path(), '/bin/ls')
self.assertEquals(code.get_execname(), '/bin/ls')
self.assertTrue(code.can_run_on(self.computer.dbcomputer))
self.assertTrue(code.can_run_on(self.computer))
othercomputer = Computer(name='another_localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida').store()
self.assertFalse(code.can_run_on(othercomputer))
def generate_ApeCalculation(codename, element, s, p, d, f):
inp = ApeCalculation.process().get_inputs_template()
inp.code = Code.get_from_string(codename)
inp._options.computer = Computer.get('localhost')
inp._options.resources = {"num_machines": 1}
check = False
for ele in ELEMENTS:
if element == ele.symbol:
nc = int(ele.number)
check = True
print check, nc
if not check:
sys.exit("element is not valid")
corpar = 1
if nc <= 2:
corpar = 0
parameters = ParameterData(
dict={
'Title': '{}'.format(element),
'CalculationMode': 'ae + pp',
'Verbose': '40',
'WaveEquation': 'schrodinger',
'XCFunctional': 'lda_x + lda_c_pz',
'NuclearCharge': '{}'.format(nc),
'PPCalculationTolerance': '1.e-6',
'PPOutputFileFormat': 'upf + siesta',
'CoreCorrection': '{}'.format(corpar),
'EigenSolverTolerance': '1.e-8',
'ODEIntTolerance': '1.e-12'
})
orbitals, PPComponents = get_orb_and_PPcomp(nc, s, p, d, f)
inp.parameters = parameters
inp.orbitals = orbitals
inp.PPComponents = PPComponents
inp._label = "{0} {1} {2} {3}".format(s, p, d, f)
inp._description = "{} pseudo generation with cut of radii in label".format(
element)
return inp
def get_transport(self):
"""
Given a computer and an aiida user (as entries of the DB) return a configured
transport to connect to the computer.
"""
from aiida.orm.computer import Computer
try:
ThisTransport = TransportFactory(self.dbcomputer.transport_type)
except MissingPluginError as e:
raise ConfigurationError(
'No transport found for {} [type {}], message: {}'.format(
self.dbcomputer.hostname, self.dbcomputer.transport_type,
e.message))
params = dict(
Computer(
dbcomputer=self.dbcomputer).get_transport_params().items() +
self.get_auth_params().items())
return ThisTransport(machine=self.dbcomputer.hostname, **params)
def setUp(self):
connec = self.__class__.connection
self.trans = connec.begin()
self.session = Session(bind=connec)
sa.session = self.session
dbcomputer = DbComputer.query.filter_by(name="localhost").first()
self.computer = Computer(dbcomputer=dbcomputer)
self.session.begin_nested()
# then each time that SAVEPOINT ends, reopen it
@event.listens_for(self.session, "after_transaction_end")
def restart_savepoint(session, transaction):
if transaction.nested and not transaction._parent.nested:
# ensure that state is expired the way
# session.commit() at the top level normally does
# (optional step)
session.expire_all()
session.begin_nested()
def get_computers_work_dir(calculations, user):
"""
Get a list of computers and their remotes working directory.
`calculations` should be a list of JobCalculation object.
"""
from aiida.orm.computer import Computer
from aiida.backends.utils import get_authinfo
computers = [Computer.get(c.dbcomputer) for c in calculations]
remotes = {}
for computer in computers:
remotes[computer.name] = {
'transport':
get_authinfo(computer=computer, aiidauser=user).get_transport(),
'computer':
computer,
}
return remotes
def setUpClass(cls):
"""
Basides the standard setup we need to add few more objects in the
database to be able to explore different requests/filters/orderings etc.
"""
# call parent setUpClass method
super(RESTApiTestCase, cls).setUpClass()
# connect the app and the api
# Init the api by connecting it the the app (N.B. respect the following
# order, api.__init__)
kwargs = dict(PREFIX=cls._url_prefix,
PERPAGE_DEFAULT=cls._PERPAGE_DEFAULT,
LIMIT_DEFAULT=cls._LIMIT_DEFAULT)
cls.app = App(__name__)
cls.app.config['TESTING'] = True
api = AiidaApi(cls.app, **kwargs)
# create test inputs
cell = ((2., 0., 0.), (0., 2., 0.), (0., 0., 2.))
structure = StructureData(cell=cell)
structure.append_atom(position=(0., 0., 0.), symbols=['Ba'])
structure.store()
cif = CifData(ase=structure.get_ase())
cif.store()
parameter1 = ParameterData(dict={"a": 1, "b": 2})
parameter1.store()
parameter2 = ParameterData(dict={"c": 3, "d": 4})
parameter2.store()
kpoint = KpointsData()
kpoint.set_kpoints_mesh([4, 4, 4])
kpoint.store()
calc = Calculation()
calc._set_attr("attr1", "OK")
calc._set_attr("attr2", "OK")
calc.store()
calc.add_link_from(structure)
calc.add_link_from(parameter1)
kpoint.add_link_from(calc, link_type=LinkType.CREATE)
calc1 = Calculation()
calc1.store()
from aiida.orm.computer import Computer
dummy_computers = [{
"name": "test1",
"hostname": "test1.epfl.ch",
"transport_type": "ssh",
"scheduler_type": "pbspro",
}, {
"name": "test2",
"hostname": "test2.epfl.ch",
"transport_type": "ssh",
"scheduler_type": "torque",
}, {
"name": "test3",
"hostname": "test3.epfl.ch",
"transport_type": "local",
"scheduler_type": "slurm",
}, {
"name": "test4",
"hostname": "test4.epfl.ch",
"transport_type": "ssh",
"scheduler_type": "slurm",
}]
for dummy_computer in dummy_computers:
computer = Computer(**dummy_computer)
computer.store()
# Prepare typical REST responses
cls.process_dummy_data()
def setUpClass(cls):
"""
Basides the standard setup we need to add few more objects in the
database to be able to explore different requests/filters/orderings etc.
"""
# call parent setUpClass method
super(RESTApiTestCase, cls).setUpClass()
# create test inputs
cell = ((2., 0., 0.), (0., 2., 0.), (0., 0., 2.))
structure = StructureData(cell=cell)
structure.append_atom(position=(0., 0., 0.), symbols=['Ba'])
structure.store()
parameter1 = ParameterData(dict={"a": 1, "b": 2})
parameter1.store()
parameter2 = ParameterData(dict={"c": 3, "d": 4})
parameter2.store()
kpoint = KpointsData()
kpoint.set_kpoints_mesh([4, 4, 4])
kpoint.store()
calc = Calculation()
calc._set_attr("attr1", "OK")
calc._set_attr("attr2", "OK")
calc.store()
calc.add_link_from(structure)
calc.add_link_from(parameter1)
kpoint.add_link_from(calc, link_type=LinkType.CREATE)
calc1 = Calculation()
calc1.store()
from aiida.orm.computer import Computer
dummy_computers = [{
"name": "test1",
"hostname": "test1.epfl.ch",
"transport_type": "ssh",
"scheduler_type": "pbspro",
}, {
"name": "test2",
"hostname": "test2.epfl.ch",
"transport_type": "ssh",
"scheduler_type": "torque",
}, {
"name": "test3",
"hostname": "test3.epfl.ch",
"transport_type": "local",
"scheduler_type": "slurm",
}, {
"name": "test4",
"hostname": "test4.epfl.ch",
"transport_type": "ssh",
"scheduler_type": "slurm",
}]
for dummy_computer in dummy_computers:
computer = Computer(**dummy_computer)
computer.store()
# Prepare typical REST responses
cls.process_dummy_data()
def computer_list(only_usable, parsable, all_comps):
"""
List available computers
"""
from aiida.orm.computer import Computer as AiiDAOrmComputer
from aiida.orm.backend import construct_backend
backend = construct_backend()
computer_names = get_computer_names()
if not parsable:
echo.echo("# List of configured computers:")
echo.echo("# (use 'verdi computer show COMPUTERNAME' "
"to see the details)")
if computer_names:
for name in sorted(computer_names):
computer = AiiDAOrmComputer.get(name)
is_configured = computer.is_user_configured(
backend.users.get_automatic_user())
is_user_enabled = computer.is_user_enabled(
backend.users.get_automatic_user())
is_usable = False # True if both enabled and configured
if not all_comps:
if not is_configured or not is_user_enabled or not computer.is_enabled(
):
continue
if computer.is_enabled():
if is_configured:
configured_str = ""
if is_user_enabled:
symbol = "*"
color = 'green'
enabled_str = ""
is_usable = True
else:
symbol = "x"
color = 'red'
enabled_str = "[DISABLED for this user]"
else:
symbol = "x"
color = 'reset'
enabled_str = ""
configured_str = " [unconfigured]"
else: # GLOBALLY DISABLED
symbol = "x"
color = 'red'
if is_configured and not is_user_enabled:
enabled_str = " [DISABLED globally AND for this user]"
else:
enabled_str = " [DISABLED globally]"
if is_configured:
configured_str = ""
else:
configured_str = " [unconfigured]"
if parsable:
echo.echo(click.style("{}".format(name), fg=color))
else:
if (not only_usable) or is_usable:
echo.echo(click.style("{} ".format(symbol), fg=color),
nl=False)
echo.echo(click.style("{} ".format(name),
bold=True,
fg=color),
nl=False)
echo.echo(
click.style("{}{}".format(enabled_str, configured_str),
fg=color))
else:
echo.echo("# No computers configured yet. Use 'verdi computer setup'")
def computer_list(self, *args):
"""
List available computers
"""
import argparse
if not is_dbenv_loaded():
load_dbenv()
from aiida.orm.computer import Computer as AiiDAOrmComputer
from aiida.backends.utils import get_automatic_user
parser = argparse.ArgumentParser(
prog=self.get_full_command_name(),
description='List the computers in the database.')
# The default states are those that are shown if no option is given
parser.add_argument(
'-C',
'--color',
action='store_true',
help="Use colors to help visualizing the different categories",
)
parser.add_argument(
'-o',
'--only-usable',
action='store_true',
help="Show only computers that are usable (i.e., "
"configured for the given user and enabled)",
)
parser.add_argument(
'-p',
'--parsable',
action='store_true',
help="Show only the computer names, one per line, "
"without any other information or string.",
)
parser.add_argument(
'-a',
'--all',
action='store_true',
help="Show also disabled or unconfigured computers",
)
parser.set_defaults(also_disabled=False)
parsed_args = parser.parse_args(args)
use_colors = parsed_args.color
only_usable = parsed_args.only_usable
parsable = parsed_args.parsable
all_comps = parsed_args.all
computer_names = self.get_computer_names()
if use_colors:
color_id = 90 # Dark gray
color_id = None # Default color
if color_id is not None:
start_color = "\x1b[{}m".format(color_id)
end_color = "\x1b[0m"
else:
start_color = ""
end_color = ""
else:
start_color = ""
end_color = ""
if not parsable:
print "{}# List of configured computers:{}".format(
start_color, end_color)
print(
"{}# (use 'verdi computer show COMPUTERNAME' "
"to see the details){}".format(start_color, end_color))
if computer_names:
for name in sorted(computer_names):
computer = AiiDAOrmComputer.get(name)
# color_id = 90 # Dark gray
# color_id = 34 # Blue
is_configured = computer.is_user_configured(
get_automatic_user())
is_user_enabled = computer.is_user_enabled(
get_automatic_user())
is_usable = False # True if both enabled and configured
if not all_comps:
if not is_configured or not is_user_enabled or not computer.is_enabled(
):
continue
if computer.is_enabled():
if is_configured:
configured_str = ""
if is_user_enabled:
symbol = "*"
color_id = None
enabled_str = ""
is_usable = True
else:
symbol = "x"
color_id = 31 # Red
enabled_str = "[DISABLED for this user]"
else:
symbol = "x"
color_id = 90 # Dark gray
enabled_str = ""
configured_str = " [unconfigured]"
else: # GLOBALLY DISABLED
symbol = "x"
color_id = 31 # Red
if is_configured and not is_user_enabled:
enabled_str = " [DISABLED globally AND for this user]"
else:
enabled_str = " [DISABLED globally]"
if is_configured:
configured_str = ""
else:
configured_str = " [unconfigured]"
if use_colors:
if color_id is not None:
start_color = "\x1b[{}m".format(color_id)
bold_sequence = "\x1b[1;{}m".format(color_id)
nobold_sequence = "\x1b[0;{}m".format(color_id)
else:
start_color = "\x1b[0m"
bold_sequence = "\x1b[1m"
nobold_sequence = "\x1b[0m"
end_color = "\x1b[0m"
else:
start_color = ""
end_color = ""
bold_sequence = ""
nobold_sequence = ""
if parsable:
print "{}{}{}".format(start_color, name, end_color)
else:
if (not only_usable) or is_usable:
print "{}{} {}{}{} {}{}{}".format(
start_color, symbol, bold_sequence, name,
nobold_sequence, enabled_str, configured_str,
end_color)
else:
print "# No computers configured yet. Use 'verdi computer setup'"
class DjangoTests(AiidaTestImplementation):
"""
Automatically takes care of the setUpClass and TearDownClass, when needed.
"""
# Note this is has to be a normal method, not a class method
def setUpClass_method(self):
self.clean_db()
self.insert_data()
def setUp_method(self):
pass
def tearDown_method(self):
pass
def insert_data(self):
"""
Insert default data into the DB.
"""
from django.core.exceptions import ObjectDoesNotExist
from aiida.backends.djsite.db.models import DbUser
from aiida.orm.computer import Computer
from aiida.common.utils import get_configured_user_email
# We create the user only once:
# Otherwise, get_automatic_user() will fail when the
# user is recreated because it caches the user!
# In any case, store it in self.user though
try:
self.user = DbUser.objects.get(email=get_configured_user_email())
except ObjectDoesNotExist:
self.user = DbUser.objects.create_user(get_configured_user_email(),
'fakepwd')
# Reqired by the calling class
self.user_email = self.user.email
# Also self.computer is required by the calling class
self.computer = Computer(name='localhost',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida')
self.computer.store()
def clean_db(self):
from aiida.backends.djsite.db.models import DbComputer
# I first delete the workflows
from aiida.backends.djsite.db.models import DbWorkflow, DbWorkflowStep, DbWorkflowData
# Complicated way to make sure we 'unwind' all the relationships
# between workflows and their children.
DbWorkflowStep.calculations.through.objects.all().delete()
DbWorkflowStep.sub_workflows.through.objects.all().delete()
DbWorkflowData.objects.all().delete()
DbWorkflowStep.objects.all().delete()
DbWorkflow.objects.all().delete()
# Delete groups
from aiida.backends.djsite.db.models import DbGroup
DbGroup.objects.all().delete()
# I first need to delete the links, because in principle I could
# not delete input nodes, only outputs. For simplicity, since
# I am deleting everything, I delete the links first
from aiida.backends.djsite.db.models import DbLink
DbLink.objects.all().delete()
# Then I delete the nodes, otherwise I cannot
# delete computers and users
from aiida.backends.djsite.db.models import DbNode
DbNode.objects.all().delete()
## I do not delete it, see discussion in setUpClass
# try:
# DbUser.objects.get(email=get_configured_user_email()).delete()
# except ObjectDoesNotExist:
# pass
DbComputer.objects.all().delete()
from aiida.backends.djsite.db.models import DbLog
DbLog.objects.all().delete()
# Note this is has to be a normal method, not a class method
def tearDownClass_method(self):
from aiida.settings import REPOSITORY_PATH
from aiida.common.setup import TEST_KEYWORD
from aiida.common.exceptions import InvalidOperation
base_repo_path = os.path.basename(os.path.normpath(REPOSITORY_PATH))
if TEST_KEYWORD not in base_repo_path:
raise InvalidOperation("Be careful. The repository for the tests "
"is not a test repository. I will not "
"empty the database and I will not delete "
"the repository. Repository path: "
"{}".format(REPOSITORY_PATH))
self.clean_db()
# I clean the test repository
shutil.rmtree(REPOSITORY_PATH, ignore_errors=True)
os.makedirs(REPOSITORY_PATH)
def get_aiida_class(self):
from aiida.orm.computer import Computer
return Computer(dbcomputer=self)
def computer_update(self, *args):
"""
Update an existing computer
"""
import argparse
from aiida.common.exceptions import NotExistent
if not is_dbenv_loaded():
load_dbenv()
# from aiida.backends.djsite.db.models import DbNode
from aiida.orm.computer import Computer
parser = argparse.ArgumentParser(prog=self.get_full_command_name(),
description='Update a computer')
# The default states are those that are shown if no option is given
parser.add_argument('computer_name', help="The name of the computer")
parsed_args = parser.parse_args(args)
computer_name = parsed_args.computer_name
try:
computer = Computer.get(computer_name)
except NotExistent:
print "No computer {} was found".format(computer_name)
sys.exit(1)
calculation_on_computer = computer.get_calculations_on_computer()
if calculation_on_computer:
# Note: this is an artificial comment.
# If you comment the following lines, you will be able to overwrite
# the old computer anyway, at your own risk.
print "You cannot modify a computer, after you run some calculations on it."
print "Disable this computer and set up a new one."
sys.exit(1)
print "*" * 75
print "WARNING! Modifying existing computer with name '{}'".format(
computer_name)
print "Are you sure you want to continue? The UUID will remain the same!"
print "Continue only if you know what you are doing."
print "If you just want to rename a computer, use the 'verdi computer rename'"
print "command. In most cases, it is better to create a new computer."
print "Moreover, if you change the transport, you must also reconfigure"
print "each computer for each user!"
print "*" * 75
print "Press [Enter] to continue, or [Ctrl]+C to exit."
raw_input()
prompt_for_computer_configuration(computer)
try:
computer.store()
except ValidationError as e:
print "Unable to store the computer: {}. Exiting...".format(
e.message)
sys.exit(1)
print "Computer '{}' successfully updated.".format(computer_name)
print "pk: {}, uuid: {}".format(computer.pk, computer.uuid)
print "(Note: machine_dependent transport parameters cannot be set via "
print "the command-line interface at the moment)"
print "OK"
pass
def deposit(what,
type,
author_name=None,
author_email=None,
url=None,
title=None,
username=None,
password=False,
user_email=None,
code_label=default_options['code'],
computer_name=None,
replace=None,
message=None,
**kwargs):
"""
Launches a
:py:class:`aiida.orm.implementation.general.calculation.job.AbstractJobCalculation`
to deposit data node to \*COD-type database.
:return: launched :py:class:`aiida.orm.implementation.general.calculation.job.AbstractJobCalculation`
instance.
:raises ValueError: if any of the required parameters are not given.
"""
from aiida.common.setup import get_property
parameters = {}
if not what:
raise ValueError("Node to be deposited is not supplied")
if not type:
raise ValueError("Deposition type is not supplied. Should be "
"one of the following: 'published', "
"'prepublication' or 'personal'")
if not username:
username = get_property('tcod.depositor_username')
if not username:
raise ValueError("Depositor username is not supplied")
if not password:
parameters['password'] = get_property('tcod.depositor_password')
if not parameters['password']:
raise ValueError("Depositor password is not supplied")
if not user_email:
user_email = get_property('tcod.depositor_email')
if not user_email:
raise ValueError("Depositor email is not supplied")
parameters['deposition-type'] = type
parameters['username'] = username
parameters['user_email'] = user_email
if type == 'published':
pass
elif type in ['prepublication', 'personal']:
if not author_name:
author_name = get_property('tcod.depositor_author_name')
if not author_name:
raise ValueError("Author name is not supplied")
if not author_email:
author_email = get_property('tcod.depositor_author_email')
if not author_email:
raise ValueError("Author email is not supplied")
if not title:
raise ValueError("Publication title is not supplied")
else:
raise ValueError("Unknown deposition type '{}' -- should be "
"one of the following: 'published', "
"'prepublication' or 'personal'".format(type))
if replace:
if str(int(replace)) != replace or int(replace) < 10000000 \
or int(replace) > 99999999:
raise ValueError("ID of the replaced structure ({}) does not "
"seem to be valid TCOD ID: must be in "
"range [10000000,99999999]".format(replace))
elif message:
raise ValueError("Message is given while the structure is not "
"redeposited -- log message is relevant to "
"redeposition only")
kwargs['additional_tags'] = {}
if title:
kwargs['additional_tags']['_publ_section_title'] = title
if author_name:
kwargs['additional_tags']['_publ_author_name'] = author_name
if replace:
kwargs['additional_tags']['_tcod_database_code'] = replace
kwargs['datablock_names'] = [replace]
cif = export_cifnode(what, store=True, **kwargs)
from aiida.orm.code import Code
from aiida.orm.computer import Computer
from aiida.orm.data.parameter import ParameterData
from aiida.common.exceptions import NotExistent
code = Code.get_from_string(code_label)
computer = None
if computer_name:
computer = Computer.get(computer_name)
calc = code.new_calc(computer=computer)
calc.set_resources({'num_machines': 1, 'num_mpiprocs_per_machine': 1})
if password:
import getpass
parameters['password'] = getpass.getpass("Password: ")
if author_name:
parameters['author_name'] = author_name
if author_email:
parameters['author_email'] = author_email
if url:
parameters['url'] = url
if replace:
parameters['replace'] = True
if message:
parameters['log-message'] = str(message)
pd = ParameterData(dict=parameters)
calc.use_cif(cif)
calc.use_parameters(pd)
calc.store_all()
calc.submit()
return calc
def computer_test(self, *args):
"""
Test the connection to a computer.
It tries to connect, to get the list of calculations on the queue and
to perform other tests.
"""
import argparse
import traceback
if not is_dbenv_loaded():
load_dbenv()
from django.core.exceptions import ObjectDoesNotExist
from aiida.common.exceptions import NotExistent
from aiida.orm.user import User
from aiida.backends.utils import get_automatic_user
from aiida.orm.computer import Computer as OrmComputer
parser = argparse.ArgumentParser(prog=self.get_full_command_name(),
description='Test a remote computer')
# The default states are those that are shown if no option is given
parser.add_argument(
'-u',
'--user',
type=str,
metavar='EMAIL',
dest='user_email',
help="Test the connection for a given AiiDA user."
"If not specified, uses the current "
"default user.",
)
parser.add_argument(
'-t',
'--traceback',
action='store_true',
help="Print the full traceback in case an exception "
"is raised",
)
parser.add_argument('computer',
type=str,
help="The name of the computer that you "
"want to test")
parsed_args = parser.parse_args(args)
user_email = parsed_args.user_email
computername = parsed_args.computer
print_traceback = parsed_args.traceback
try:
computer = self.get_computer(name=computername)
except NotExistent:
print >> sys.stderr, "No computer exists with name '{}'".format(
computername)
sys.exit(1)
if user_email is None:
user = User(dbuser=get_automatic_user())
else:
user_list = User.search_for_users(email=user_email)
# If no user is found
if not user_list:
print >> sys.stderr, ("No user with email '{}' in the "
"database.".format(user_email))
sys.exit(1)
user = user_list[0]
print "Testing computer '{}' for user {}...".format(
computername, user.email)
try:
dbauthinfo = computer.get_dbauthinfo(user._dbuser)
except NotExistent:
print >> sys.stderr, ("User with email '{}' is not yet configured "
"for computer '{}' yet.".format(
user.email, computername))
sys.exit(1)
warning_string = None
if not dbauthinfo.enabled:
warning_string = (
"** NOTE! Computer is disabled for the "
"specified user!\n Do you really want to test it? [y/N] ")
if not computer.is_enabled():
warning_string = ("** NOTE! Computer is disabled!\n"
" Do you really want to test it? [y/N] ")
if warning_string:
answer = raw_input(warning_string)
if not (answer == 'y' or answer == 'Y'):
sys.exit(0)
s = OrmComputer(dbcomputer=dbauthinfo.dbcomputer).get_scheduler()
t = dbauthinfo.get_transport()
## STARTING TESTS HERE
num_failures = 0
num_tests = 0
try:
print "> Testing connection..."
with t:
s.set_transport(t)
num_tests += 1
for test in [
self._computer_test_get_jobs,
self._computer_create_temp_file
]:
num_tests += 1
try:
succeeded = test(transport=t,
scheduler=s,
dbauthinfo=dbauthinfo)
except Exception as e:
print "* The test raised an exception!"
if print_traceback:
print "** Full traceback:"
# Indent
print "\n".join([
" {}".format(l)
for l in traceback.format_exc().splitlines()
])
else:
print "** {}: {}".format(e.__class__.__name__,
e.message)
print(
"** (use the --traceback option to see the "
"full traceback)")
succeeded = False
if not succeeded:
num_failures += 1
if num_failures:
print "Some tests failed! ({} out of {} failed)".format(
num_failures, num_tests)
else:
print "Test completed (all {} tests succeeded)".format(
num_tests)
except Exception as e:
print "** Error while trying to connect to the computer! I cannot "
print " perform following tests, so I stop."
if print_traceback:
print "** Full traceback:"
# Indent
print "\n".join([
" {}".format(l)
for l in traceback.format_exc().splitlines()
])
else:
print "{}: {}".format(e.__class__.__name__, e.message)
print(
"(use the --traceback option to see the "
"full traceback)")
succeeded = False
def setUp(self):
"""
"""
from aiida import work
from aiida.orm.code import Code
from aiida.orm.nodes.parameter import Dict
from aiida.orm.nodes.structure import StructureData
from aiida.orm.nodes.remote import RemoteData
from ase.spacegroup import crystal
from aiida_quantumespresso.calculations.pw import PwCalculation
from aiida_yambo.calculations.gw import YamboCalculation
from aiida.common.links import LinkType
from aiida.orm.computer import Computer as AiidaOrmComputer
from aiida.common.datastructures import calc_states
from aiida.plugins.utils import DataFactory
runner = work.Runner(poll_interval=0.,
rmq_config=None,
enable_persistence=None)
work.set_runner(runner)
self.computer = AiidaOrmComputer(name="testcase")
# conf_attrs hostname, description, enabled_state, transport_type, scheduler_type, workdir
# mpirun_command , default_mpiprocs_per_machine,
self.computer._set_hostname_string("localhost")
self.computer._set_enabled_state_string('True')
self.computer._set_transport_type_string("local")
self.computer._set_scheduler_type_string("direct")
self.computer._set_workdir_string("/tmp/testcase/{username}/base")
self.computer.store()
create_authinfo(computer=self.computer).store()
self.code_yambo = Code()
self.code_yambo.label = "yambo"
os_env = os.environ.copy()
yambo_path = subprocess.check_output(['which', 'mock_yambo'],
env=os_env).strip()
self.code_yambo.set_remote_computer_exec((self.computer, yambo_path))
self.code_yambo.set_input_plugin_name('yambo.yambo')
self.code_p2y = Code()
self.code_p2y.label = "p2y"
p2y_path = subprocess.check_output(['which', 'mock_p2y'],
env=os_env).strip()
self.code_p2y.set_remote_computer_exec((self.computer, p2y_path))
self.code_p2y.set_input_plugin_name('yambo.yambo')
self.code_yambo.store()
self.code_p2y.store()
self.calc_pw = PwCalculation()
self.calc_pw.set_computer(self.computer)
self.calc_pw.set_resources({
"num_machines": 1,
"num_mpiprocs_per_machine": 16,
'default_mpiprocs_per_machine': 16
})
StructureData = DataFactory('structure')
cell = [[15.8753100000, 0.0000000000, 0.0000000000],
[0.0000000000, 15.8753100000, 0.0000000000],
[0.0000000000, 0.0000000000, 2.4696584760]]
s = StructureData(cell=cell)
self.calc_pw.use_structure(s)
print((self.calc_pw.store_all(), " pw calc"))
pw_remote_folder = RemoteData(computer=self.computer,
remote_path="/tmp/testcase/work/calcPW")
print((pw_remote_folder.store(), "pw remote data"))
self.calc_pw._set_state(calc_states.PARSING)
pw_remote_folder.add_link_from(self.calc_pw,
label='remote_folder',
link_type=LinkType.CREATE)
outputs = Dict(
dict={
"lsda": False,
"number_of_bands": 80,
"number_of_electrons": 8.0,
"number_of_k_points": 147,
"non_colinear_calculation": False
})
outputs.store()
outputs.add_link_from(self.calc_pw,
label='output_parameters',
link_type=LinkType.CREATE)
self.calc = YamboCalculation()
self.calc.set_computer(self.computer)
self.calc.use_code(self.code_p2y)
p2y_settings = {
u'ADDITIONAL_RETRIEVE_LIST':
[u'r-*', u'o-*', u'l-*', u'l_*', u'LOG/l-*_CPU_1'],
u'INITIALISE':
True
}
yambo_settings = {
u'ADDITIONAL_RETRIEVE_LIST':
[u'r-*', u'o-*', u'l-*', u'l_*', u'LOG/l-*_CPU_1']
}
self.calc.use_settings(Dict(dict=p2y_settings))
self.calc.set_resources({
"num_machines": 1,
"num_mpiprocs_per_machine": 16,
'default_mpiprocs_per_machine': 16
})
self.calc.use_parent_calculation(self.calc_pw)
print((self.calc.store_all(), " yambo calc"))
self.calc._set_state(calc_states.PARSING)
a = 5.388
cell = crystal('Si', [(0, 0, 0)],
spacegroup=227,
cellpar=[a, a, a, 90, 90, 90],
primitive_cell=True)
self.struc = StructureData(ase=cell)
self.struc.store()
self.parameters = Dict(
dict={
"BndsRnXp": [1.0, 48.0],
"Chimod": "Hartree",
"DysSolver": "n",
"FFTGvecs": 25,
"FFTGvecs_units": "Ry",
"GbndRnge": [1.0, 48.0],
"HF_and_locXC": True,
"LongDrXp": [1.0, 0.0, 0.0],
"NGsBlkXp": 2,
"NGsBlkXp_units": "Ry",
"QPkrange": [[1, 145, 3, 5]],
"SE_CPU": "1 2 4",
"SE_ROLEs": "q qp b",
"X_all_q_CPU": "1 1 4 2",
"X_all_q_ROLEs": "q k c v",
"em1d": True,
"gw0": True,
"ppa": True,
"rim_cut": True
})
self.yambo_settings = Dict(
dict={
"ADDITIONAL_RETRIEVE_LIST": [
"r-*", "o-*", "l-*", "l_*", "LOG/l-*_CPU_1",
"aiida/ndb.QP", "aiida/ndb.HF_and_locXC"
]
})
self.p2y_settings = Dict(
dict={
"ADDITIONAL_RETRIEVE_LIST": [
'r-*', 'o-*', 'l-*', 'l_*', 'LOG/l-*_CPU_1',
'aiida/ndb.QP', 'aiida/ndb.HF_and_locXC'
],
'INITIALISE':
True
})
self.yambo_calc_set = Dict(
dict={
'resources': {
"num_machines": 1,
"num_mpiprocs_per_machine": 16
},
'max_wallclock_seconds': 60 * 29,
'max_memory_kb': 1 * 88 * 1000000,
"queue_name":
"s3parvc3", #'custom_scheduler_commands': u"#PBS -A Pra14_3622" ,
'environment_variables': {
"OMP_NUM_THREADS": "1"
}
})
self.p2y_calc_set = Dict(
dict={
'resources': {
"num_machines": 1,
"num_mpiprocs_per_machine": 2
},
'max_wallclock_seconds': 60 * 2,
'max_memory_kb': 1 * 10 * 1000000,
"queue_name":
"s3parvc3", # 'custom_scheduler_commands': u"#PBS -A Pra14_3622" ,
'environment_variables': {
"OMP_NUM_THREADS": "2"
}
})
self.remote_folder = RemoteData(computer=self.computer,
remote_path="/tmp/testcase/work/calcX")
self.remote_folder.store()
self.remote_folder.add_link_from(self.calc,
label='remote_folder',
link_type=LinkType.CREATE)
self.calc._set_state(calc_states.FINISHED)
