def workflow_list(short, all_states, depth, past_days, workflows):
"""List legacy workflows"""
from aiida.backends.utils import get_workflow_list
from aiida.orm.workflow import get_workflow_info
from aiida.orm.backend import construct_backend # pylint: disable=no-name-in-module
tab_size = 2
backend = construct_backend()
current_user = backend.users.get_automatic_user()
wf_list = get_workflow_list([workflow.pk for workflow in workflows],
user=current_user,
all_states=all_states,
n_days_ago=past_days)
for workflow in wf_list:
if not workflow.is_subworkflow() or workflow in workflows:
echo.echo('\n'.join(
get_workflow_info(workflow,
tab_size=tab_size,
short=short,
depth=depth)))
if not workflows:
if all_states:
echo.echo_info('# No workflows found')
else:
echo.echo_info('# No running workflows found')
def update(comment, node, comment_id):
"""
Update a comment.
id = The id of the comment
comment = The comment (a string) to be added to the node(s)
"""
from aiida.orm.backend import construct_backend
backend = construct_backend()
user = backend.users.get_automatic_user()
# read the comment from terminal if it is not on command line
if comment is None:
try:
current_comment = node.get_comments(comment_id)[0]
except IndexError:
echo.echo_error(
"Comment with id '{}' not found".format(comment_id))
return 1
comment = multi_line_input.edit_comment(current_comment['content'])
# pylint: disable=protected-access
node._update_comment(comment, comment_id, user)
return 0
def get_transport(self, user=None):
"""
Return a Tranport class, configured with all correct parameters.
The Transport is closed (meaning that if you want to run any operation with
it, you have to open it first (i.e., e.g. for a SSH tranport, you have
to open a connection). To do this you can call ``transport.open()``, or simply
run within a ``with`` statement::
transport = Computer.get_transport()
with transport:
print(transport.whoami())
:param user: if None, try to obtain a transport for the default user.
Otherwise, pass a valid User.
:return: a (closed) Transport, already configured with the connection
parameters to the supercomputer, as configured with ``verdi computer configure``
for the user specified as a parameter ``user``.
"""
from aiida.orm.backend import construct_backend
backend = construct_backend()
if user is None:
authinfo = backend.authinfos.get(self, backend.users.get_automatic_user())
else:
authinfo = backend.authinfos.get(self, user)
transport = authinfo.get_transport()
return transport
def create_authinfo(computer, store=False):
"""
Allow the current user to use the given computer.
Deal with backwards compatibility down to aiida 0.11
"""
from aiida.orm import backend as orm_backend
authinfo = None
if hasattr(orm_backend, 'construct_backend'):
backend = orm_backend.construct_backend()
authinfo = backend.authinfos.create(computer=computer, user=get_current_user())
if store:
authinfo.store()
else:
from aiida.backends.settings import BACKEND
from aiida.backends.profile import BACKEND_SQLA, BACKEND_DJANGO
if BACKEND == BACKEND_DJANGO:
from aiida.backends.djsite.db.models import DbAuthInfo
authinfo = DbAuthInfo(dbcomputer=computer.dbcomputer, aiidauser=get_current_user())
elif BACKEND == BACKEND_SQLA:
from aiida.backends.sqlalchemy.models.authinfo import DbAuthInfo
from aiida.backends.sqlalchemy import get_scoped_session
_ = get_scoped_session()
authinfo = DbAuthInfo(dbcomputer=computer.dbcomputer, aiidauser=get_current_user())
if store:
authinfo.save()
return authinfo
def test_workflow_info(self):
"""
This test checks that the workflow info is generate without any
exceptions
:return:
"""
backend = construct_backend()
# Assuming there is only one user
user = backend.users.find(email=self.user_email)[0]
# Creating a simple workflow & storing it
a = WFTestEmpty()
a.store()
# Emulate the workflow list
for w in get_workflow_list(all_states=True, user=user):
if not w.is_subworkflow():
get_workflow_info(w)
# Create a workflow with sub-workflows and store it
b = WFTestSimpleWithSubWF()
b.store()
# Emulate the workflow list
for w in get_workflow_list(all_states=True, user=user):
if not w.is_subworkflow():
get_workflow_info(w)
# Start the first workflow and perform a workflow list
b.start()
for w in get_workflow_list(all_states=True, user=user):
if not w.is_subworkflow():
get_workflow_info(w)
def configure_computer(computer, user=None, **kwargs):
"""Configure a computer for a user with valid auth params passed via kwargs."""
from aiida.orm.backend import construct_backend
transport_cls = computer.get_transport_class()
backend = construct_backend()
user = user or backend.users.get_automatic_user()
try:
authinfo = computer.get_authinfo(user)
except NotExistent:
authinfo = backend.authinfos.create(computer, user)
auth_params = authinfo.get_auth_params()
valid_keys = set(transport_cls.get_valid_auth_params())
if not set(kwargs.keys()).issubset(valid_keys):
invalid_keys = [key for key in kwargs if key not in valid_keys]
raise ValueError('{transport}: recieved invalid authentication parameter(s) "{invalid}"'.format(
transport=transport_cls, invalid=invalid_keys))
if valid_keys:
auth_params.update(kwargs)
authinfo.set_auth_params(auth_params)
from aiida.settings import BACKEND
if BACKEND == 'sqlalchemy':
authinfo._dbauthinfo.auth_params = auth_params # pylint: disable=protected-access
authinfo.store()
def get_automatic_user():
try:
from aiida.backends.utils import get_automatic_user
automatic_user = get_automatic_user()
except ImportError:
from aiida.orm.backend import construct_backend
backend = construct_backend()
automatic_user = backend.users.get_automatic_user()
return automatic_user
def _backend(self):
"""
Get the backend
"""
if self.__backend is None:
# Lazy load the backend so we don't do it too early (i.e. before load_dbenv())
from aiida.orm.backend import construct_backend
self.__backend = construct_backend()
return self.__backend
def kill_all():
from aiida.orm.backend import construct_backend
backend = construct_backend()
automatic_user = backend.users.get_automatic_user()
w_list = DbWorkflow.query.filter(
DbWorkflow.user == automatic_user.dbuser,
DbWorkflow.state != wf_states.FINISHED).all()
for w in w_list:
Workflow.get_subclass_from_uuid(w.uuid).kill()
def calculation_cleanworkdir(calculations, past_days, older_than, computers,
force):
"""
Clean all content of all output remote folders of calculations.
If no explicit calculations are specified as arguments, one or both of the -p and -o options has to be specified.
If both are specified, a logical AND is done between the two, i.e. the calculations that will be cleaned have been
modified AFTER [-p option] days from now, but BEFORE [-o option] days from now.
"""
from aiida.orm.backend import construct_backend
from aiida.orm.utils.loaders import ComputerEntityLoader, IdentifierType
from aiida.orm.utils.remote import clean_remote, get_calculation_remote_paths
if calculations:
if (past_days is not None and older_than is not None):
echo.echo_critical(
'specify either explicit calculations or use the filtering options'
)
else:
if (past_days is None and older_than is None):
echo.echo_critical(
'if no explicit calculations are specified, at least one filtering option is required'
)
calculations_pks = [calculation.pk for calculation in calculations]
path_mapping = get_calculation_remote_paths(calculations_pks, past_days,
older_than, computers)
if path_mapping is None:
echo.echo_critical('no calculations found with the given criteria')
if not force:
path_count = sum(
[len(paths) for computer, paths in path_mapping.items()])
warning = 'Are you sure you want to clean the work directory of {} calculations?'.format(
path_count)
click.confirm(warning, abort=True)
backend = construct_backend()
user = backend.users.get_automatic_user()
for computer_uuid, paths in path_mapping.items():
counter = 0
computer = ComputerEntityLoader.load_entity(
computer_uuid, identifier_type=IdentifierType.UUID)
transport = backend.authinfos.get(computer, user).get_transport()
with transport:
for path in paths:
clean_remote(transport, path)
counter += 1
echo.echo_success('{} remote folders cleaned on {}'.format(
counter, computer.name))
def kill_all():
from aiida.backends.djsite.db.models import DbWorkflow
from aiida.orm.backend import construct_backend
backend = construct_backend()
q_object = Q(user=backend.users.get_automatic_user().id)
q_object.add(~Q(state=wf_states.FINISHED), Q.AND)
w_list = DbWorkflow.objects.filter(q_object)
for w in w_list:
Workflow.get_subclass_from_uuid(w.uuid).kill()
def get_current_user():
"""Get current user backwards compatibly with aiida-core <= 0.12.1."""
current_user = None
if backend_obj_users():
from aiida.orm.backend import construct_backend # pylint: disable=no-name-in-module
backend = construct_backend()
current_user = backend.users.get_automatic_user()
else:
from aiida.backends.utils import get_automatic_user # pylint: disable=no-name-in-module
current_user = get_automatic_user()
return current_user
def get_computer(name=TEST_COMPUTER, workdir=None, configure=False):
"""Get local computer.
Sets up local computer with 'name' or reads it from database,
if it exists.
:param name: Name of local computer
:param workdir: path to work directory (required if creating a new computer)
:param configure: whether to congfigure a new computer for the user email
:return: The computer node
:rtype: :py:class:`aiida.orm.Computer`
"""
from aiida.common.exceptions import NotExistent
if aiida_version() >= cmp_version("1.0.0a2"):
from aiida.orm.backend import construct_backend
backend = construct_backend()
get_computer = lambda name: backend.computers.get(name=name)
create_computer = backend.computers.create
else:
from aiida.orm import Computer
get_computer = Computer.get
create_computer = Computer
try:
computer = get_computer(name)
except NotExistent:
if workdir is None:
raise ValueError(
"to create a new computer, a work directory must be supplied")
computer = create_computer(
name=name,
description='localhost computer set up by aiida_crystal17 tests',
hostname=name,
workdir=workdir,
transport_type='local',
scheduler_type='direct',
enabled_state=True)
computer.store()
if configure:
try:
# aiida-core v1
from aiida.control.computer import configure_computer
configure_computer(computer)
except ImportError:
configure_computer_v012(computer)
return computer
def get_report_messages(pk, depth, levelname):
"""Return list of log messages with given levelname and their depth for a node with a given pk."""
backend = construct_backend()
filters = {
'objpk': pk,
}
entries = backend.logs.find(filter_by=filters)
entries = [
entry for entry in entries
if LOG_LEVELS[entry.levelname] >= LOG_LEVELS[levelname]
]
return [(_, depth) for _ in entries]
def setUpClass(cls, *args, **kwargs):
from aiida.orm.backend import construct_backend
# Note: this will raise an exception, that will be seen as a test
# failure. To be safe, you should do the same check also in the tearDownClass
# to avoid that it is run
check_if_tests_can_run()
cls.__backend_instance = cls.get_backend_class()()
cls.__backend_instance.setUpClass_method(*args, **kwargs)
cls.backend = construct_backend()
cls._class_was_setup = True
def complete(self, ctx, incomplete): # pylint: disable=unused-argument,no-self-use
"""
Return possible completions based on an incomplete value
:returns: list of tuples of valid entry points (matching incomplete) and a description
"""
from aiida.orm.backend import construct_backend
backend = construct_backend()
users = backend.users.find()
return [(user.email, '') for user in users
if user.email.startswith(incomplete)]
def setUpClass(cls):
super(TestVerdiDataRemote, cls).setUpClass()
new_comp = Computer(name='comp',
hostname='localhost',
transport_type='local',
scheduler_type='direct',
workdir=tempfile.mkdtemp())
new_comp.store()
b = construct_backend()
aiidauser = b.users.get_automatic_user()
authinfo = DbAuthInfo(dbcomputer=new_comp.dbcomputer,
aiidauser=aiidauser.dbuser)
authinfo.save()
def setUp(self):
super(TestBackendLog, self).setUp()
self._backend = construct_backend()
self._record = {
'time': now(),
'loggername': 'loggername',
'levelname': logging.getLevelName(LOG_LEVEL_REPORT),
'objname': 'objname',
'objpk': 0,
'message': 'This is a template record message',
'metadata': {
'content': 'test'
},
}
def setUp(self):
"""Prepare current user and computer builder with common properties."""
from aiida.orm.backend import construct_backend
from aiida.scheduler import SchedulerFactory
backend = construct_backend()
self.comp_builder = ComputerBuilder(label='test', description='Test Computer', enabled=True, hostname='localhost')
self.comp_builder.scheduler = 'direct'
self.comp_builder.work_dir = '/tmp/aiida'
self.comp_builder.prepend_text = ''
self.comp_builder.append_text = ''
self.comp_builder.mpiprocs_per_machine = 8
self.comp_builder.mpirun_command = 'mpirun'
self.comp_builder.shebang = '#!xonsh'
self.user = backend.users.get_automatic_user()
def configure_computer_main(computer, user, **kwargs):
"""Configure a computer via the CLI."""
from aiida.orm.backend import construct_backend
from aiida.control.computer import configure_computer
from aiida.common.utils import get_configured_user_email
backend = construct_backend()
user = user or backend.users.get_automatic_user()
echo.echo_info('Configuring computer {} for user {}.'.format(computer.name, user.email))
if user.email != get_configured_user_email():
echo.echo_info('Configuring different user, defaults may not be appropriate.')
configure_computer(computer, user=user, **kwargs)
echo.echo_success('{} successfully configured for {}'.format(computer.name, user.email))
def get_computer_configuration(computer, user=None):
"""Get the configuratio of computer for user as a dictionary."""
from aiida.orm.backend import construct_backend
backend = construct_backend()
user = user or backend.users.get_automatic_user()
config = {}
try:
authinfo = backend.authinfos.get(computer, user)
config = authinfo.get_auth_params()
except NotExistent:
pass
return config
def __init__(self, **kwargs):
from aiida.orm.backend import construct_backend
super(Group, self).__init__()
self._backend = construct_backend()
given_dbgroup = kwargs.pop('dbgroup', None)
if given_dbgroup is not None:
# Check that there is no other parameter passed besides dbgroup
if kwargs:
raise ValueError("If you pass a dbgroups, you cannot pass any "
"further parameter")
if isinstance(given_dbgroup, (int, long)):
dbgroup_res = DbGroup.query.filter_by(id=given_dbgroup).first()
if not dbgroup_res:
raise NotExistent("Group with pk={} does not exist".format(
given_dbgroup))
dbgroup = dbgroup_res
elif isinstance(given_dbgroup, DbGroup):
dbgroup = given_dbgroup
else:
raise ValueError("Unrecognised inputs '{}'".format(kwargs))
else:
name = kwargs.pop('name', None)
if name is None:
raise ValueError("You have to specify a group name")
group_type = kwargs.pop('type_string',
"") # By default, an user group
# Get the user and extract the dbuser instance
user = kwargs.pop('user', self._backend.users.get_automatic_user())
user = user.dbuser
description = kwargs.pop('description', "")
if kwargs:
raise ValueError("Too many parameters passed to Group, the "
"unknown parameters are: {}".format(", ".join(
kwargs.keys())))
dbgroup = DbGroup(name=name,
description=description,
user=user,
type=group_type)
self._dbgroup = utils.ModelWrapper(dbgroup)
def convert(self, value, param, ctx):
from aiida.orm.backend import construct_backend
backend = construct_backend()
results = backend.users.find(email=value)
if not results:
if self._create:
return backend.users.create(email=value)
else:
self.fail("User '{}' not found".format(value), param, ctx)
elif len(results) > 1:
self.fail("Multiple users found with email '{}': {}".format(
value, results))
return results[0]
def get_default(ctx):
"""Determine the default value from the context."""
from aiida.orm.backend import construct_backend
backend = construct_backend()
user = ctx.params['user'] or backend.users.get_automatic_user()
computer = ctx.params['computer']
try:
authinfo = backend.authinfos.get(computer=computer, user=user)
except NotExistent:
authinfo = backend.authinfos.create(computer=computer, user=user)
non_interactive = ctx.params['non_interactive']
old_authparams = authinfo.get_auth_params()
if not also_noninteractive and non_interactive:
raise click.MissingParameter()
suggestion = old_authparams.get(key)
suggestion = suggestion or transport_option_default(key, computer)
return suggestion
def add(comment, nodes):
"""
Add comment to one or more nodes in the database
"""
from aiida.orm.backend import construct_backend
backend = construct_backend()
user = backend.users.get_automatic_user()
if not comment:
comment = multi_line_input.edit_comment()
for node in nodes:
node.add_comment(comment, user)
echo.echo_info("Comment added to node(s) '{}'".format(", ".join(
[str(node.pk) for node in nodes])))
def create_authinfo(computer):
"""
Allow the current user to use the given computer.
Deal with backwards compatibility down to aiida 0.11
"""
from aiida import load_profile
load_profile()
from aiida.orm import backend as orm_backend
authinfo = None
if hasattr(orm_backend, 'construct_backend'):
backend = orm_backend.construct_backend()
authinfo = backend.authinfos.create(computer=computer,
user=get_current_user())
else:
from aiida.backends.djsite.db.models import DbAuthInfo
authinfo = DbAuthInfo(dbcomputer=computer.dbcomputer,
aiidauser=get_current_user())
return authinfo
def __init__(self, **kwargs):
"""
Initializes the Workflow super class, store the instance in the DB and in case
stores the starting parameters.
If initialized with an uuid the Workflow is loaded from the DB, if not a new
workflow is generated and added to the DB following the stack frameworks. This
means that only modules inside aiida.workflows are allowed to implements
the workflow super calls and be stored. The caller names, modules and files are
retrieved from the stack.
:param uuid: a string with the uuid of the object to be loaded.
:param params: a dictionary of storable objects to initialize the specific workflow
:raise: NotExistent: if there is no entry of the desired workflow kind with
the given uuid.
"""
super(AbstractWorkflow, self).__init__()
self._backend = construct_backend()
def setUp(self):
"""Prepare computer builder with common properties."""
from aiida.orm.backend import construct_backend
from aiida.control.computer import ComputerBuilder
self.backend = construct_backend()
self.user = self.backend.users.get_automatic_user()
self.runner = CliRunner()
self.comp_builder = ComputerBuilder(label='test_comp_setup')
self.comp_builder.hostname = 'localhost'
self.comp_builder.description = 'Test Computer'
self.comp_builder.enabled = True
self.comp_builder.scheduler = 'direct'
self.comp_builder.work_dir = '/tmp/aiida'
self.comp_builder.prepend_text = ''
self.comp_builder.append_text = ''
self.comp_builder.mpiprocs_per_machine = 8
self.comp_builder.mpirun_command = 'mpirun'
self.comp_builder.shebang = '#!xonsh'
def __init__(self, **kwargs):
from aiida.backends.djsite.db.models import DbGroup
from aiida.orm.backend import construct_backend
self._backend = construct_backend()
dbgroup = kwargs.pop('dbgroup', None)
if dbgroup is not None:
if kwargs:
raise ValueError(
"If you pass a dbgroups, you cannot pass any further parameter"
)
if isinstance(dbgroup, (int, long)):
try:
dbgroup = DbGroup.objects.get(pk=dbgroup)
except ObjectDoesNotExist:
raise NotExistent(
"Group with pk={} does not exist".format(dbgroup))
type_check(dbgroup, DbGroup)
self._dbgroup = utils.ModelWrapper(dbgroup)
else:
name = kwargs.pop('name', None)
if name is None:
raise ValueError("You have to specify a group name")
group_type = kwargs.pop('type_string',
"") # By default, a user group
# Get the user and extract dbuser instance
user = kwargs.pop('user', self._backend.users.get_automatic_user())
description = kwargs.pop('description', "")
self._dbgroup = utils.ModelWrapper(
DbGroup(name=name,
description=description,
user=user.dbuser,
type=group_type))
if kwargs:
raise ValueError("Too many parameters passed to Group, the "
"unknown parameters are: {}".format(", ".join(
kwargs.keys())))
def query(datatype, project, past_days, group_pks, all_users):
"""
Perform the query
"""
import datetime
from aiida.orm.implementation import Group
from aiida.orm.user import User
from aiida.orm.backend import construct_backend
from aiida.orm.querybuilder import QueryBuilder
from aiida.utils import timezone
backend = construct_backend()
qbl = QueryBuilder()
if all_users is False:
user = backend.users.get_automatic_user()
qbl.append(User, tag="creator", filters={"email": user.email})
else:
qbl.append(User, tag="creator")
# If there is a time restriction
data_filters = {}
if past_days is not None:
now = timezone.now()
n_days_ago = now - datetime.timedelta(days=past_days)
data_filters.update({"ctime": {'>=': n_days_ago}})
qbl.append(datatype, tag="data", created_by="creator", filters=data_filters, project=project)
# If there is a group restriction
if group_pks is not None:
group_filters = dict()
group_filters.update({"id": {"in": group_pks}})
qbl.append(Group, tag="group", filters=group_filters, group_of="data")
qbl.order_by({datatype: {'ctime': 'asc'}})
object_list = qbl.distinct()
return object_list.all()
